Skip to content

Commit 0016f54

Browse files
andrii-suseandrii-suse
committed
Factory: Add support for repos without Media1 suffix
1 parent 1e4fabf commit 0016f54

34 files changed

+269
-201
lines changed

‎script/cfg.py‎

Lines changed: 91 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
read_files_isos = '''rsync -4 --list-only $rsync_pwd_option PRODUCTISOPATH/ | grep -P 'Media1?.iso$' | grep -E 'ARCHORS' | awk '{ $1=$2=$3=$4=""; print substr($0,5); }' >> __envsub/files_iso.lst
2323
'''
2424

25-
read_files_repo = '''rsync -4 --list-only $rsync_pwd_option PRODUCTREPOPATH/ | grep -P 'Media[1-3](.license)?$' | awk '{ $1=$2=$3=$4=""; print substr($0,5); } ' | grep -v IGNOREPATTERN | grep -E 'REPOORS' | grep -E 'ARCHORS' >> __envsub/files_repo.lst
25+
read_files_repo = '''rsync -4 --list-only $rsync_pwd_option PRODUCTREPOPATH/ | grep '^d' | grep -P 'Media[1-3](.license)?$' | awk '{ $1=$2=$3=$4=""; print substr($0,5); } ' | grep -v IGNOREPATTERN | grep -E 'REPOORS' | grep -E 'ARCHORS' >> __envsub/files_repo.lst
2626
'''
2727

2828
read_files_repo_media = '''rsync -4 $rsync_pwd_option PRODUCTREPOPATH/*Media1/media.1/media __envsub/Media1.lst'''
@@ -105,7 +105,7 @@ def rsync_commands(checksum):
105105
echo "rsync --timeout=3600 -tlp4 --specials PRODUCTISOPATH/$folder/$src /var/lib/openqa/factory/$asset/"
106106
echo "rsync --timeout=3600 -tlp4 --specials PRODUCTISOPATH/$folder/$src.sha256 /var/lib/openqa/factory/other/"
107107
echo ""
108-
done < <(grep ${arch} __envsub/files_iso.lst | sort)
108+
done < <(grep ${arch} __envsub/files_iso.lst | LANG=C_ALL sort)
109109
done
110110
done'''
111111

@@ -154,7 +154,7 @@ def pre_rsync_repo(repos):
154154
[ ! -f __envsub/files_iso.lst ] || buildid=$(cat __envsub/files_iso.lst | grep $arch | grep -E 'FLAVORORS' | grep -o -E '(Build|Snapshot)[^-]*' | head -n 1)
155155
[ -n "$buildid" ] || buildid=$(grep -hEo 'Build[0-9]+(.[0-9]+)?' __envsub/Media1_*.lst 2>/dev/null | head -n 1)
156156
if [[ ! -n $build ]] && [[ ! $buildid =~ (Build|Snapshot)[0-9]*[0-9]$ ]]; then
157-
buildid=${buildid%%\.[[:alpha:]]*}
157+
buildid=${buildid%%\\.[[:alpha:]]*}
158158
fi
159159
160160
while read src; do
@@ -165,7 +165,7 @@ def pre_rsync_repo(repos):
165165
destSuffix=${dest#$destPrefix}
166166
mid=''
167167
dest=$destPrefix$mid$destSuffix
168-
[[ ! $src =~ .*\.license ]] || [[ $dest == *license* ]] || dest=$dest.license'''
168+
[[ ! $src =~ .*\\.license ]] || [[ $dest == *license* ]] || dest=$dest.license'''
169169

170170
def rsync_repodir2():
171171
return r'''
@@ -210,9 +210,64 @@ def rsync_repomultiarch(destpath, debug, source):
210210
[ ! -z "$src" ] || continue
211211
[[ ''' + dest + ''' =~ $arch ]] || [[ ''' + dest + ''' =~ ${arch//i686/i586} ]] || [[ "ARCHITECTURREPO" == . ]] || break
212212
dest=''' + dest + '''
213-
[[ ! $src =~ .*\.license ]] || dest=$dest.license'''
213+
[[ ! $src =~ .*\\.license ]] || dest=$dest.license'''
214214

215215

216+
def rsync_repodir1_dest_media0(dest, debug, source, folder):
217+
repo = os.path.basename(folder).lstrip("*")
218+
xtra=""
219+
res = r'''
220+
221+
# archs=(ARCHITECTURREPO)
222+
buildid=$(cat __envsub/files_iso.lst | grep -E 'FLAVORORS' | grep -o -E '(Build|Snapshot)[^-]*' | head -n 1)
223+
224+
# for arch in "${archs[@]}"; do
225+
while read src; do
226+
[ ! -z "$src" ] || continue
227+
[[ ''' + dest + ''' =~ $arch ]] || [[ ''' + dest + ''' =~ ${arch//i686/i586} ]] || [[ "ARCHITECTURREPO" == . ]] || break
228+
dest=''' + dest + '''
229+
[[ ! $src =~ .*\\.license ]] || dest=$dest.license
230+
dest1=""
231+
[[ $src != *".license" ]] || dest1=.license
232+
'''
233+
234+
res = res + '''
235+
[[ $src != *"-Debug" ]] || {
236+
'''
237+
if debug:
238+
xtra="--include=" + debug + " --exclude={aarch64,armv7hl,i586,i686,noarch,nosrc,ppc64,ppc64le,s390x,src,x86_64}/*"
239+
res = res + '''
240+
echo rsync --timeout=3600 -rtlp4 --delete --specials ''' + xtra + ''' PRODUCTREPOPATH/''' + folder + '''/$src/ /var/lib/openqa/factory/repo/$dest-CURRENT-Debug/
241+
echo rsync --timeout=3600 -rtlp4 --delete --specials --link-dest /var/lib/openqa/factory/repo/$dest-CURRENT-Debug/ /var/lib/openqa/factory/repo/$dest-CURRENT-Debug/ /var/lib/openqa/factory/repo/$dest-$buildid-Debug
242+
'''
243+
res = res + '''
244+
continue
245+
}
246+
'''
247+
248+
res = res + '''
249+
[[ $src != *"-Source" ]] || {
250+
'''
251+
if source:
252+
xtra="--include=" + source + " --exclude={aarch64,armv7hl,i586,i686,noarch,nosrc,ppc64,ppc64le,s390x,src,x86_64}/*"
253+
res = res + '''
254+
echo rsync --timeout=3600 -rtlp4 --delete --specials ''' + xtra + ''' PRODUCTREPOPATH/''' + folder + '''/$src/ /var/lib/openqa/factory/repo/$dest-CURRENT-Source/
255+
echo rsync --timeout=3600 -rtlp4 --delete --specials --link-dest /var/lib/openqa/factory/repo/$dest-CURRENT-Source/ /var/lib/openqa/factory/repo/$dest-CURRENT-Source/ /var/lib/openqa/factory/repo/$dest-$buildid-Source
256+
'''
257+
res = res + '''
258+
continue
259+
}
260+
'''
261+
262+
res = res + '''
263+
echo rsync --timeout=3600 -rtlp4 --delete --specials PRODUCTREPOPATH/''' + folder + '''/$src/ /var/lib/openqa/factory/repo/$dest-CURRENT$dest1/
264+
echo rsync --timeout=3600 -rtlp4 --delete --specials --link-dest /var/lib/openqa/factory/repo/$dest-CURRENT$dest1/ /var/lib/openqa/factory/repo/$dest-CURRENT$dest1/ /var/lib/openqa/factory/repo/$dest-$buildid$dest1
265+
done < <(LANG=C_ALL sort __envsub/files_repo_''' + repo + '''.lst )
266+
# done
267+
'''
268+
269+
return res
270+
216271

217272
def openqa_call_fix_destiso(distri, version, staging):
218273
if not staging:
@@ -288,6 +343,7 @@ def pre_openqa_call_start(repos):
288343
''' + openqa_call_news(news, news_archs) + '''
289344
}
290345
fi
346+
# test "$destiso" != "" || continue
291347
echo "''' + openqa_cli + ''' \\\\\"
292348
(
293349
echo \" DISTRI=$distri \\\\
@@ -351,7 +407,7 @@ def openqa_call_start_ex(checksum):
351407
echo " HDD_$n=$src \\"
352408
echo " CHECKSUM_HDD_$n=\$(cut -b-64 /var/lib/openqa/factory/other/$src.sha256 | grep -E '[0-9a-f]{5,40}' | head -n1) \\"
353409
fi
354-
done < <(grep ${arch} __envsub/files_iso.lst | sort)
410+
done < <(grep ${arch} __envsub/files_iso.lst | LANG=C_ALL sort)
355411
'''
356412

357413
# if MIRROREPO is set - expressions for FLAVORASREPOORS will evaluate to false
@@ -430,16 +486,34 @@ def openqa_call_extra(repos):
430486
[[ $repo =~ license ]] || echo " REPO_REPOPREFIX$repoKey=$repoDest \\\\"
431487
''' + openqa_call_extra(repos) + '''
432488
: $((i++))
433-
done < <(grep $repot$additional_repo_suffix __envsub/files_repo.lst | grep REPOTYPE | grep $arch | sort)
489+
done < <(grep $repot$additional_repo_suffix __envsub/files_repo.lst | grep REPOTYPE | grep $arch | LANG=C_ALL sort)
434490
done'''
435491

436-
def openqa_call_repot1_debugsource():
437-
return '''[[ $src != *Media2* ]] || repoKey=${repoKey}_DEBUGINFO
492+
def openqa_call_repot1_debugsource(debug, source):
493+
res = '''[[ $src != *Media2* ]] || repoKey=${repoKey}_DEBUGINFO
438494
[[ $src != *Media2* ]] || dest=$dest-debuginfo
439495
[[ $src != *Media3* ]] || repoKey=${repoKey}_SOURCE
440496
[[ $src != *Media3* ]] || dest=$dest-source'''
497+
if debug:
498+
res = res + '''
499+
[[ $src != *-Debug ]] || repoKey=${repoKey}_DEBUG
500+
[[ $src != *-Debug ]] || dest=$dest-Debug'''
501+
else:
502+
res = res + '''
503+
[[ $src != *-Debug ]] || continue'''
504+
505+
if source:
506+
res = res + '''
507+
[[ $src != *-Source ]] || repoKey=${repoKey}_SOURCE
508+
[[ $src != *-Source ]] || dest=$dest-Source'''
509+
else:
510+
res = res + '''
511+
[[ $src != *-Source ]] || continue'''
512+
513+
return res
514+
441515

442-
openqa_call_repot1 = lambda: '''
516+
openqa_call_repot1 = lambda debug, source: '''
443517
while read src; do
444518
dest=$src
445519
dest=${dest%-Build*}
@@ -459,7 +533,7 @@ def openqa_call_repot1_debugsource():
459533
repoKey=${repoKey//-/_}
460534
repoKey=${repoKey//./_}
461535
repoKey=${repoKey//$/}
462-
''' + openqa_call_repot1_debugsource() + '''
536+
''' + openqa_call_repot1_debugsource(debug, source) + '''
463537
dest=${dest//-Media1/}
464538
dest=${dest//-Media2/}
465539
dest=${dest//-Media3/}
@@ -469,19 +543,20 @@ def openqa_call_repot1_debugsource():
469543
echo " REPO_$i=$dest \\\\"
470544
[[ $src =~ license ]] || echo " REPO_$repoKey=$dest \\\\"
471545
[[ ! $repoKey =~ _DEBUGINFO ]] || [ -z "DEBUG_PACKAGES" ] || echo " REPO_${{repoKey}}_PACKAGES='DEBUG_PACKAGES' \\\\"
546+
[[ $repoKey != *_DEBUG ]] || [ -z "DEBUG_PACKAGES" ] || echo " REPO_${{repoKey}}_PACKAGES='DEBUG_PACKAGES' \\\\"
472547
[[ ! $repoKey =~ _SOURCE ]] || [ -z "SOURCE_PACKAGES" ] || echo " REPO_${{repoKey}}_PACKAGES='SOURCE_PACKAGES' \\\\"
473548
: $((i++))
474-
done < <(grep ${{arch//i686/i586}} __envsub/files_repo.lst {} | sort)'''
549+
done < <(grep ${{arch//i686/i586}} __envsub/files_repo.lst {} | LANG=C_ALL sort)'''
475550

476551

477-
openqa_call_repot1_dest = lambda dest: '''
552+
openqa_call_repot1_dest = lambda dest, debug, source: '''
478553
while read src; do
479554
[[ ''' + dest + ''' =~ $arch ]] || [[ ''' + dest + ''' =~ ${arch//i686/i586} ]] || [[ "ARCHITECTURREPO" == . ]] || break
480555
dest=''' + dest + r'''
481556
[[ ! $src =~ .*\.license ]] || dest=$dest.license
482557
dest=$dest-$buildex
483558
repoKey=REPOKEY
484-
''' + openqa_call_repot1_debugsource() + '''
559+
''' + openqa_call_repot1_debugsource(debug, source) + '''
485560
repoKey=${repoKey^^}
486561
repoKey=${repoKey//-/_}
487562
repoKey=${repoKey//./_}
@@ -508,14 +583,14 @@ def openqa_call_end(version):
508583
[ $flavor != MicroOS-DVD ] || flavor=DVD
509584
[ $flavor != Staging-MicroOS-DVD ] || flavor=Staging-DVD
510585
echo " FLAVOR=${flavor//Tumbleweed-/} \\\\"
511-
) | LC_COLLATE=C sort
586+
) | LANG=C_ALL sort
512587
echo ""
513588
done
514589
done
515590
'''
516591
return '''
517592
echo " FLAVOR=$flavor \\\\"
518-
) | LC_COLLATE=C sort
593+
) | LANG=C_ALL sort
519594
echo ""
520595
done
521596
done

‎script/scriptgen.py‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -659,7 +659,7 @@ def gen_read_files(self, f):
659659
suffix = ""
660660
if repodir.attrib.get("suffix"):
661661
suffix = repodir.attrib["suffix"]
662-
self.p(
662+
args = (
663663
cfg.read_files_repo,
664664
f,
665665
txt,
@@ -671,6 +671,9 @@ def gen_read_files(self, f):
671671
"ARCHORS",
672672
archs.replace(" ", "|").replace("armv7hl", "armv7hl|armv7l"),
673673
)
674+
if self.media1 == "0":
675+
args += ("| grep -P 'Media[1-3](.license)?$'", "")
676+
self.p(*args)
674677

675678
# let's sync media.1/media to be able verify build_id
676679
if "ToTest" or "LEO" in self.ag.envdir or self.version_from_media:
@@ -904,6 +907,14 @@ def gen_print_rsync_repo(self, f):
904907
continue
905908
if not r.attrib.get("dest", ""):
906909
self.p(cfg.rsync_repodir1, f, "mid=''", "mid='{}'".format(r.attrib.get("mid", "")))
910+
elif self.media1 == "0":
911+
self.p(
912+
cfg.rsync_repodir1_dest_media0(
913+
r.attrib["dest"], r.get("debug", ""), r.get("source", ""), r.attrib["folder"]
914+
),
915+
f,
916+
)
917+
continue
907918
elif not r.attrib.get("gen", ""):
908919
self.p(cfg.rsync_repodir1_dest(r.attrib["dest"]), f)
909920

@@ -1156,15 +1167,20 @@ def gen_print_openqa(self, f):
11561167
for r in repodirs:
11571168
if r.attrib.get("dest", "") == "":
11581169
self.p(
1159-
cfg.openqa_call_repot1(),
1170+
cfg.openqa_call_repot1(r.get("debug", ""), r.get("source", "")),
11601171
f,
11611172
"REPOKEY",
11621173
r.attrib.get("rename", r.tag),
11631174
"mid=''",
11641175
"mid='{}'".format(r.attrib.get("mid", "")),
11651176
)
11661177
else:
1167-
self.p(cfg.openqa_call_repot1_dest(r.attrib["dest"]), f, "REPOKEY", r.attrib.get("rename", r.tag))
1178+
self.p(
1179+
cfg.openqa_call_repot1_dest(r.attrib["dest"], r.get("debug", ""), r.get("source", "")),
1180+
f,
1181+
"REPOKEY",
1182+
r.attrib.get("rename", r.tag),
1183+
)
11681184

11691185
for ren in self.renames:
11701186
self.p(" dest=${{dest//{}/{}}}".format(ren[0], ren[1]), f)
@@ -1173,7 +1189,7 @@ def gen_print_openqa(self, f):
11731189
" [ $i != 0 ] || {{ {}; }}".format(cfg.openqa_call_repo0()), f, "REPO0_ISO", "$dest", f
11741190
)
11751191
media_filter = ""
1176-
if r.attrib.get("debug", "") == "" or r.attrib.get("source", "") == "":
1192+
if self.media1 != "0" and (r.attrib.get("debug", "") == "" or r.attrib.get("source", "") == ""):
11771193
if r.attrib.get("debug", "") == "" and r.attrib.get("source", "") == "":
11781194
media_filter = "| grep Media1 "
11791195
elif r.attrib.get("debug", "") == "":

‎t/obs/openSUSE:Factory:ToTest/base/files_repo_0NonFree.lst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
openSUSE-Addon-NonOss-20250210-x86_64-Debug
2+
openSUSE-Addon-NonOss-20250210-x86_64-Source
3+
openSUSE-Addon-NonOss-20250210-x86_64

‎t/obs/openSUSE:Factory:ToTest/base/files_repo_0openSUSE.lst‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
openSUSE-20250210-x86_64.license
2+
openSUSE-20250210-x86_64-Debug
3+
openSUSE-20250210-x86_64-Source
4+
openSUSE-20250210-x86_64

‎t/obs/openSUSE:Factory:ToTest/base/files_repo_openSUSE-Addon-NonOss-ftp-ftp.lst‎

Lines changed: 0 additions & 2 deletions
This file was deleted.

‎t/obs/openSUSE:Factory:ToTest/base/files_repo_openSUSE-ftp-ftp.lst‎

Lines changed: 0 additions & 4 deletions
This file was deleted.

‎t/obs/openSUSE:Factory:ToTest/base/print_openqa.before‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -11,15 +11,15 @@
1111
MIRROR_HTTPS=https://openqa.opensuse.org/assets/repo/openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210 \
1212
MIRROR_PREFIX=http://openqa.opensuse.org/assets/repo \
1313
REPO_0=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210 \
14-
REPO_1=openSUSE-Tumbleweed-oss-x86_64.license-Snapshot20250210 \
15-
REPO_2=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-debuginfo \
16-
REPO_3=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-source \
14+
REPO_1=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-Debug \
15+
REPO_2=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-Source \
16+
REPO_3=openSUSE-Tumbleweed-oss-x86_64.license-Snapshot20250210 \
1717
REPO_4=openSUSE-Tumbleweed-non-oss-x86_64-Snapshot20250210 \
1818
REPO_NON_OSS=openSUSE-Tumbleweed-non-oss-x86_64-Snapshot20250210 \
1919
REPO_OSS=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210 \
20-
REPO_OSS_DEBUGINFO=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-debuginfo \
21-
REPO_OSS_DEBUGINFO_PACKAGES='java*,kernel-default-debug*,kernel-default-base-debug*,mraa-debug*,wicked-debug*' \
22-
REPO_OSS_SOURCE=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-source \
20+
REPO_OSS_DEBUG=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-Debug \
21+
REPO_OSS_DEBUG_PACKAGES='java*,kernel-default-debug*,kernel-default-base-debug*,mraa-debug*,wicked-debug*' \
22+
REPO_OSS_SOURCE=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-Source \
2323
REPO_OSS_SOURCE_PACKAGES='coreutils*,yast2-network*,yast2-http-server*' \
2424
SUSEMIRROR=http://openqa.opensuse.org/assets/repo/openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210 \
2525
VERSION=Tumbleweed \
@@ -42,15 +42,15 @@
4242
MIRROR_HTTPS=https://openqa.opensuse.org/assets/repo/openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210 \
4343
MIRROR_PREFIX=http://openqa.opensuse.org/assets/repo \
4444
REPO_0=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210 \
45-
REPO_1=openSUSE-Tumbleweed-oss-x86_64.license-Snapshot20250210 \
46-
REPO_2=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-debuginfo \
47-
REPO_3=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-source \
45+
REPO_1=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-Debug \
46+
REPO_2=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-Source \
47+
REPO_3=openSUSE-Tumbleweed-oss-x86_64.license-Snapshot20250210 \
4848
REPO_4=openSUSE-Tumbleweed-non-oss-x86_64-Snapshot20250210 \
4949
REPO_NON_OSS=openSUSE-Tumbleweed-non-oss-x86_64-Snapshot20250210 \
5050
REPO_OSS=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210 \
51-
REPO_OSS_DEBUGINFO=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-debuginfo \
52-
REPO_OSS_DEBUGINFO_PACKAGES='java*,kernel-default-debug*,kernel-default-base-debug*,mraa-debug*,wicked-debug*' \
53-
REPO_OSS_SOURCE=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-source \
51+
REPO_OSS_DEBUG=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-Debug \
52+
REPO_OSS_DEBUG_PACKAGES='java*,kernel-default-debug*,kernel-default-base-debug*,mraa-debug*,wicked-debug*' \
53+
REPO_OSS_SOURCE=openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210-Source \
5454
REPO_OSS_SOURCE_PACKAGES='coreutils*,yast2-network*,yast2-http-server*' \
5555
SUSEMIRROR=http://openqa.opensuse.org/assets/repo/openSUSE-Tumbleweed-oss-x86_64-Snapshot20250210 \
5656
VERSION=Tumbleweed \

0 commit comments

Comments
 (0)