Skip to content

Commit 0e30f3d

Browse files
andrii-suseandrii-suse
committed
Factory: Add support for repos without Media1 suffix
1 parent 1e4fabf commit 0e30f3d

34 files changed

+269
-203
lines changed

‎script/cfg.py‎

Lines changed: 89 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
read_files_isos = '''rsync -4 --list-only $rsync_pwd_option PRODUCTISOPATH/ | grep -P 'Media1?.iso$' | grep -E 'ARCHORS' | awk '{ $1=$2=$3=$4=""; print substr($0,5); }' >> __envsub/files_iso.lst
2323
'''
2424

25-
read_files_repo = '''rsync -4 --list-only $rsync_pwd_option PRODUCTREPOPATH/ | grep -P 'Media[1-3](.license)?$' | awk '{ $1=$2=$3=$4=""; print substr($0,5); } ' | grep -v IGNOREPATTERN | grep -E 'REPOORS' | grep -E 'ARCHORS' >> __envsub/files_repo.lst
25+
read_files_repo = '''rsync -4 --list-only $rsync_pwd_option PRODUCTREPOPATH/ | grep '^d' | grep -P 'Media[1-3](.license)?$' | awk '{ $1=$2=$3=$4=""; print substr($0,5); } ' | grep -v IGNOREPATTERN | grep -E 'REPOORS' | grep -E 'ARCHORS' >> __envsub/files_repo.lst
2626
'''
2727

2828
read_files_repo_media = '''rsync -4 $rsync_pwd_option PRODUCTREPOPATH/*Media1/media.1/media __envsub/Media1.lst'''
@@ -105,7 +105,7 @@ def rsync_commands(checksum):
105105
echo "rsync --timeout=3600 -tlp4 --specials PRODUCTISOPATH/$folder/$src /var/lib/openqa/factory/$asset/"
106106
echo "rsync --timeout=3600 -tlp4 --specials PRODUCTISOPATH/$folder/$src.sha256 /var/lib/openqa/factory/other/"
107107
echo ""
108-
done < <(grep ${arch} __envsub/files_iso.lst | sort)
108+
done < <(grep ${arch} __envsub/files_iso.lst | LANG=C.UTF-8 sort)
109109
done
110110
done'''
111111

@@ -154,7 +154,7 @@ def pre_rsync_repo(repos):
154154
[ ! -f __envsub/files_iso.lst ] || buildid=$(cat __envsub/files_iso.lst | grep $arch | grep -E 'FLAVORORS' | grep -o -E '(Build|Snapshot)[^-]*' | head -n 1)
155155
[ -n "$buildid" ] || buildid=$(grep -hEo 'Build[0-9]+(.[0-9]+)?' __envsub/Media1_*.lst 2>/dev/null | head -n 1)
156156
if [[ ! -n $build ]] && [[ ! $buildid =~ (Build|Snapshot)[0-9]*[0-9]$ ]]; then
157-
buildid=${buildid%%\.[[:alpha:]]*}
157+
buildid=${buildid%%\\.[[:alpha:]]*}
158158
fi
159159
160160
while read src; do
@@ -165,7 +165,7 @@ def pre_rsync_repo(repos):
165165
destSuffix=${dest#$destPrefix}
166166
mid=''
167167
dest=$destPrefix$mid$destSuffix
168-
[[ ! $src =~ .*\.license ]] || [[ $dest == *license* ]] || dest=$dest.license'''
168+
[[ ! $src =~ .*\\.license ]] || [[ $dest == *license* ]] || dest=$dest.license'''
169169

170170
def rsync_repodir2():
171171
return r'''
@@ -210,9 +210,62 @@ def rsync_repomultiarch(destpath, debug, source):
210210
[ ! -z "$src" ] || continue
211211
[[ ''' + dest + ''' =~ $arch ]] || [[ ''' + dest + ''' =~ ${arch//i686/i586} ]] || [[ "ARCHITECTURREPO" == . ]] || break
212212
dest=''' + dest + '''
213-
[[ ! $src =~ .*\.license ]] || dest=$dest.license'''
213+
[[ ! $src =~ .*\\.license ]] || dest=$dest.license'''
214214

215215

216+
def rsync_repodir1_dest_media0(dest, debug, source, folder):
217+
repo = os.path.basename(folder).lstrip("*")
218+
xtra=""
219+
res = r'''
220+
221+
# archs=(ARCHITECTURREPO)
222+
buildid=$(cat __envsub/files_iso.lst | grep -E 'FLAVORORS' | grep -o -E '(Build|Snapshot)[^-]*' | head -n 1)
223+
224+
# for arch in "${archs[@]}"; do
225+
while read src; do
226+
[ ! -z "$src" ] || continue
227+
[[ ''' + dest + ''' =~ $arch ]] || [[ ''' + dest + ''' =~ ${arch//i686/i586} ]] || [[ "ARCHITECTURREPO" == . ]] || break
228+
dest=''' + dest + '''
229+
[[ ! $src =~ .*\\.license ]] || dest=$dest.license
230+
'''
231+
232+
res = res + '''
233+
[[ $src != *"-Debug" ]] || {
234+
'''
235+
if debug:
236+
xtra="--include=" + debug + " --exclude={aarch64,armv7hl,i586,i686,noarch,nosrc,ppc64,ppc64le,s390x,src,x86_64}/*"
237+
res = res + '''
238+
echo rsync --timeout=3600 -rtlp4 --delete --specials ''' + xtra + ''' PRODUCTREPOPATH/''' + folder + '''/$src/ /var/lib/openqa/factory/repo/$dest-CURRENT-Debug/
239+
echo rsync --timeout=3600 -rtlp4 --delete --specials --link-dest /var/lib/openqa/factory/repo/$dest-CURRENT-Debug/ /var/lib/openqa/factory/repo/$dest-CURRENT-Debug/ /var/lib/openqa/factory/repo/$dest-$buildid-Debug
240+
'''
241+
res = res + '''
242+
continue
243+
}
244+
'''
245+
246+
res = res + '''
247+
[[ $src != *"-Source" ]] || {
248+
'''
249+
if source:
250+
xtra="--include=" + source + " --exclude={aarch64,armv7hl,i586,i686,noarch,nosrc,ppc64,ppc64le,s390x,src,x86_64}/*"
251+
res = res + '''
252+
echo rsync --timeout=3600 -rtlp4 --delete --specials ''' + xtra + ''' PRODUCTREPOPATH/''' + folder + '''/$src/ /var/lib/openqa/factory/repo/$dest-CURRENT-Source/
253+
echo rsync --timeout=3600 -rtlp4 --delete --specials --link-dest /var/lib/openqa/factory/repo/$dest-CURRENT-Source/ /var/lib/openqa/factory/repo/$dest-CURRENT-Source/ /var/lib/openqa/factory/repo/$dest-$buildid-Source
254+
'''
255+
res = res + '''
256+
continue
257+
}
258+
'''
259+
260+
res = res + '''
261+
echo rsync --timeout=3600 -rtlp4 --delete --specials PRODUCTREPOPATH/''' + folder + '''/$src/ /var/lib/openqa/factory/repo/$dest-CURRENT/
262+
echo rsync --timeout=3600 -rtlp4 --delete --specials --link-dest /var/lib/openqa/factory/repo/$dest-CURRENT/ /var/lib/openqa/factory/repo/$dest-CURRENT/ /var/lib/openqa/factory/repo/$dest-$buildid
263+
done < <(LANG=C.UTF-8 sort __envsub/files_repo_''' + repo + '''.lst )
264+
# done
265+
'''
266+
267+
return res
268+
216269

217270
def openqa_call_fix_destiso(distri, version, staging):
218271
if not staging:
@@ -288,6 +341,7 @@ def pre_openqa_call_start(repos):
288341
''' + openqa_call_news(news, news_archs) + '''
289342
}
290343
fi
344+
# test "$destiso" != "" || continue
291345
echo "''' + openqa_cli + ''' \\\\\"
292346
(
293347
echo \" DISTRI=$distri \\\\
@@ -351,7 +405,7 @@ def openqa_call_start_ex(checksum):
351405
echo " HDD_$n=$src \\"
352406
echo " CHECKSUM_HDD_$n=\$(cut -b-64 /var/lib/openqa/factory/other/$src.sha256 | grep -E '[0-9a-f]{5,40}' | head -n1) \\"
353407
fi
354-
done < <(grep ${arch} __envsub/files_iso.lst | sort)
408+
done < <(grep ${arch} __envsub/files_iso.lst | LANG=C.UTF-8 sort)
355409
'''
356410

357411
# if MIRROREPO is set - expressions for FLAVORASREPOORS will evaluate to false
@@ -430,16 +484,34 @@ def openqa_call_extra(repos):
430484
[[ $repo =~ license ]] || echo " REPO_REPOPREFIX$repoKey=$repoDest \\\\"
431485
''' + openqa_call_extra(repos) + '''
432486
: $((i++))
433-
done < <(grep $repot$additional_repo_suffix __envsub/files_repo.lst | grep REPOTYPE | grep $arch | sort)
487+
done < <(grep $repot$additional_repo_suffix __envsub/files_repo.lst | grep REPOTYPE | grep $arch | LANG=C.UTF-8 sort)
434488
done'''
435489

436-
def openqa_call_repot1_debugsource():
437-
return '''[[ $src != *Media2* ]] || repoKey=${repoKey}_DEBUGINFO
490+
def openqa_call_repot1_debugsource(debug, source):
491+
res = '''[[ $src != *Media2* ]] || repoKey=${repoKey}_DEBUGINFO
438492
[[ $src != *Media2* ]] || dest=$dest-debuginfo
439493
[[ $src != *Media3* ]] || repoKey=${repoKey}_SOURCE
440494
[[ $src != *Media3* ]] || dest=$dest-source'''
495+
if debug:
496+
res = res + '''
497+
[[ $src != *-Debug ]] || repoKey=${repoKey}_DEBUG
498+
[[ $src != *-Debug ]] || dest=$dest-Debug'''
499+
else:
500+
res = res + '''
501+
[[ $src != *-Debug ]] || continue'''
502+
503+
if source:
504+
res = res + '''
505+
[[ $src != *-Source ]] || repoKey=${repoKey}_SOURCE
506+
[[ $src != *-Source ]] || dest=$dest-Source'''
507+
else:
508+
res = res + '''
509+
[[ $src != *-Source ]] || continue'''
510+
511+
return res
512+
441513

442-
openqa_call_repot1 = lambda: '''
514+
openqa_call_repot1 = lambda debug, source: '''
443515
while read src; do
444516
dest=$src
445517
dest=${dest%-Build*}
@@ -459,7 +531,7 @@ def openqa_call_repot1_debugsource():
459531
repoKey=${repoKey//-/_}
460532
repoKey=${repoKey//./_}
461533
repoKey=${repoKey//$/}
462-
''' + openqa_call_repot1_debugsource() + '''
534+
''' + openqa_call_repot1_debugsource(debug, source) + '''
463535
dest=${dest//-Media1/}
464536
dest=${dest//-Media2/}
465537
dest=${dest//-Media3/}
@@ -469,19 +541,20 @@ def openqa_call_repot1_debugsource():
469541
echo " REPO_$i=$dest \\\\"
470542
[[ $src =~ license ]] || echo " REPO_$repoKey=$dest \\\\"
471543
[[ ! $repoKey =~ _DEBUGINFO ]] || [ -z "DEBUG_PACKAGES" ] || echo " REPO_${{repoKey}}_PACKAGES='DEBUG_PACKAGES' \\\\"
544+
[[ $repoKey != *_DEBUG ]] || [ -z "DEBUG_PACKAGES" ] || echo " REPO_${{repoKey}}_PACKAGES='DEBUG_PACKAGES' \\\\"
472545
[[ ! $repoKey =~ _SOURCE ]] || [ -z "SOURCE_PACKAGES" ] || echo " REPO_${{repoKey}}_PACKAGES='SOURCE_PACKAGES' \\\\"
473546
: $((i++))
474-
done < <(grep ${{arch//i686/i586}} __envsub/files_repo.lst {} | sort)'''
547+
done < <(grep ${{arch//i686/i586}} __envsub/files_repo.lst {} | LANG=C.UTF-8 sort)'''
475548

476549

477-
openqa_call_repot1_dest = lambda dest: '''
550+
openqa_call_repot1_dest = lambda dest, debug, source: '''
478551
while read src; do
479552
[[ ''' + dest + ''' =~ $arch ]] || [[ ''' + dest + ''' =~ ${arch//i686/i586} ]] || [[ "ARCHITECTURREPO" == . ]] || break
480553
dest=''' + dest + r'''
481554
[[ ! $src =~ .*\.license ]] || dest=$dest.license
482555
dest=$dest-$buildex
483556
repoKey=REPOKEY
484-
''' + openqa_call_repot1_debugsource() + '''
557+
''' + openqa_call_repot1_debugsource(debug, source) + '''
485558
repoKey=${repoKey^^}
486559
repoKey=${repoKey//-/_}
487560
repoKey=${repoKey//./_}
@@ -508,14 +581,14 @@ def openqa_call_end(version):
508581
[ $flavor != MicroOS-DVD ] || flavor=DVD
509582
[ $flavor != Staging-MicroOS-DVD ] || flavor=Staging-DVD
510583
echo " FLAVOR=${flavor//Tumbleweed-/} \\\\"
511-
) | LC_COLLATE=C sort
584+
) | LANG=C.UTF-8 sort
512585
echo ""
513586
done
514587
done
515588
'''
516589
return '''
517590
echo " FLAVOR=$flavor \\\\"
518-
) | LC_COLLATE=C sort
591+
) | LANG=C.UTF-8 sort
519592
echo ""
520593
done
521594
done

‎script/scriptgen.py‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -659,7 +659,7 @@ def gen_read_files(self, f):
659659
suffix = ""
660660
if repodir.attrib.get("suffix"):
661661
suffix = repodir.attrib["suffix"]
662-
self.p(
662+
args = (
663663
cfg.read_files_repo,
664664
f,
665665
txt,
@@ -671,6 +671,9 @@ def gen_read_files(self, f):
671671
"ARCHORS",
672672
archs.replace(" ", "|").replace("armv7hl", "armv7hl|armv7l"),
673673
)
674+
if self.media1 == "0":
675+
args += ("| grep -P 'Media[1-3](.license)?$'", "")
676+
self.p(*args)
674677

675678
# let's sync media.1/media to be able verify build_id
676679
if "ToTest" or "LEO" in self.ag.envdir or self.version_from_media:
@@ -803,7 +806,7 @@ def gen_print_rsync_assets(self, f):
803806
break
804807
done
805808
echo "rsync --timeout=3600 -tlp4 --specials PRODUCTPATH/$folder/*$src /var/lib/openqa/factory/other/"
806-
done < <(sort __envsub/files_asset.lst)""",
809+
done < <(LANG=C.UTF-8 sort __envsub/files_asset.lst)""",
807810
f,
808811
)
809812

@@ -904,6 +907,14 @@ def gen_print_rsync_repo(self, f):
904907
continue
905908
if not r.attrib.get("dest", ""):
906909
self.p(cfg.rsync_repodir1, f, "mid=''", "mid='{}'".format(r.attrib.get("mid", "")))
910+
elif self.media1 == "0":
911+
self.p(
912+
cfg.rsync_repodir1_dest_media0(
913+
r.attrib["dest"], r.get("debug", ""), r.get("source", ""), r.attrib["folder"]
914+
),
915+
f,
916+
)
917+
continue
907918
elif not r.attrib.get("gen", ""):
908919
self.p(cfg.rsync_repodir1_dest(r.attrib["dest"]), f)
909920

@@ -1132,7 +1143,7 @@ def gen_print_openqa(self, f):
11321143
done
11331144
[ -z "$asset_tag" ] || echo " ASSET_${asset_tag^^}=$src \\\\"
11341145
: $((i++))
1135-
done < <(grep ${arch} __envsub/files_asset.lst | sort)""",
1146+
done < <(grep ${arch} __envsub/files_asset.lst | LANG=C.UTF-8 sort)""",
11361147
f,
11371148
)
11381149

@@ -1156,15 +1167,20 @@ def gen_print_openqa(self, f):
11561167
for r in repodirs:
11571168
if r.attrib.get("dest", "") == "":
11581169
self.p(
1159-
cfg.openqa_call_repot1(),
1170+
cfg.openqa_call_repot1(r.get("debug", ""), r.get("source", "")),
11601171
f,
11611172
"REPOKEY",
11621173
r.attrib.get("rename", r.tag),
11631174
"mid=''",
11641175
"mid='{}'".format(r.attrib.get("mid", "")),
11651176
)
11661177
else:
1167-
self.p(cfg.openqa_call_repot1_dest(r.attrib["dest"]), f, "REPOKEY", r.attrib.get("rename", r.tag))
1178+
self.p(
1179+
cfg.openqa_call_repot1_dest(r.attrib["dest"], r.get("debug", ""), r.get("source", "")),
1180+
f,
1181+
"REPOKEY",
1182+
r.attrib.get("rename", r.tag),
1183+
)
11681184

11691185
for ren in self.renames:
11701186
self.p(" dest=${{dest//{}/{}}}".format(ren[0], ren[1]), f)
@@ -1173,7 +1189,7 @@ def gen_print_openqa(self, f):
11731189
" [ $i != 0 ] || {{ {}; }}".format(cfg.openqa_call_repo0()), f, "REPO0_ISO", "$dest", f
11741190
)
11751191
media_filter = ""
1176-
if r.attrib.get("debug", "") == "" or r.attrib.get("source", "") == "":
1192+
if self.media1 != "0" and (r.attrib.get("debug", "") == "" or r.attrib.get("source", "") == ""):
11771193
if r.attrib.get("debug", "") == "" and r.attrib.get("source", "") == "":
11781194
media_filter = "| grep Media1 "
11791195
elif r.attrib.get("debug", "") == "":

‎t/obs/openSUSE:Factory:ToTest/base/files_repo_0NonFree.lst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
openSUSE-Addon-NonOss-20250210-x86_64-Debug
2+
openSUSE-Addon-NonOss-20250210-x86_64-Source
3+
openSUSE-Addon-NonOss-20250210-x86_64

‎t/obs/openSUSE:Factory:ToTest/base/files_repo_0openSUSE.lst‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
openSUSE-20250210-x86_64.license
2+
openSUSE-20250210-x86_64-Debug
3+
openSUSE-20250210-x86_64-Source
4+
openSUSE-20250210-x86_64

‎t/obs/openSUSE:Factory:ToTest/base/files_repo_openSUSE-Addon-NonOss-ftp-ftp.lst‎

Lines changed: 0 additions & 2 deletions
This file was deleted.

‎t/obs/openSUSE:Factory:ToTest/base/files_repo_openSUSE-ftp-ftp.lst‎

Lines changed: 0 additions & 4 deletions
This file was deleted.

‎t/obs/openSUSE:Factory:ToTest/base/print_openqa.before‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -11,15 +11,15 @@
1111
MIRROR_HTTPS=https://openqa.opensuse.org/assets/repo/openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210 \
1212
MIRROR_PREFIX=http://openqa.opensuse.org/assets/repo \
1313
REPO_0=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210 \
14-
REPO_1=openSUSE-Tumbleweed-oss-x86_64.license-Snapshot20250210 \
15-
REPO_2=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-debuginfo \
16-
REPO_3=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-source \
14+
REPO_1=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-Debug \
15+
REPO_2=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-Source \
16+
REPO_3=openSUSE-Tumbleweed-oss-x86_64.license-Snapshot20250210 \
1717
REPO_4=openSUSE-Tumbleweed-non-oss-x86_64-Snapshot20250210 \
1818
REPO_NON_OSS=openSUSE-Tumbleweed-non-oss-x86_64-Snapshot20250210 \
1919
REPO_OSS=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210 \
20-
REPO_OSS_DEBUGINFO=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-debuginfo \
21-
REPO_OSS_DEBUGINFO_PACKAGES='java*,kernel-default-debug*,kernel-default-base-debug*,mraa-debug*,wicked-debug*' \
22-
REPO_OSS_SOURCE=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-source \
20+
REPO_OSS_DEBUG=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-Debug \
21+
REPO_OSS_DEBUG_PACKAGES='java*,kernel-default-debug*,kernel-default-base-debug*,mraa-debug*,wicked-debug*' \
22+
REPO_OSS_SOURCE=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-Source \
2323
REPO_OSS_SOURCE_PACKAGES='coreutils*,yast2-network*,yast2-http-server*' \
2424
SUSEMIRROR=http://openqa.opensuse.org/assets/repo/openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210 \
2525
VERSION=Tumbleweed \
@@ -42,15 +42,15 @@
4242
MIRROR_HTTPS=https://openqa.opensuse.org/assets/repo/openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210 \
4343
MIRROR_PREFIX=http://openqa.opensuse.org/assets/repo \
4444
REPO_0=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210 \
45-
REPO_1=openSUSE-Tumbleweed-oss-x86_64.license-Snapshot20250210 \
46-
REPO_2=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-debuginfo \
47-
REPO_3=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-source \
45+
REPO_1=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-Debug \
46+
REPO_2=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-Source \
47+
REPO_3=openSUSE-Tumbleweed-oss-x86_64.license-Snapshot20250210 \
4848
REPO_4=openSUSE-Tumbleweed-non-oss-x86_64-Snapshot20250210 \
4949
REPO_NON_OSS=openSUSE-Tumbleweed-non-oss-x86_64-Snapshot20250210 \
5050
REPO_OSS=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210 \
51-
REPO_OSS_DEBUGINFO=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-debuginfo \
52-
REPO_OSS_DEBUGINFO_PACKAGES='java*,kernel-default-debug*,kernel-default-base-debug*,mraa-debug*,wicked-debug*' \
53-
REPO_OSS_SOURCE=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-source \
51+
REPO_OSS_DEBUG=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-Debug \
52+
REPO_OSS_DEBUG_PACKAGES='java*,kernel-default-debug*,kernel-default-base-debug*,mraa-debug*,wicked-debug*' \
53+
REPO_OSS_SOURCE=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-Source \
5454
REPO_OSS_SOURCE_PACKAGES='coreutils*,yast2-network*,yast2-http-server*' \
5555
SUSEMIRROR=http://openqa.opensuse.org/assets/repo/openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210 \
5656
VERSION=Tumbleweed \

0 commit comments

Comments
 (0)