Skip to content

Commit bb89d9d

Browse files
minlearnminlearnminlearnminlearn
committed
githubactions-pushed
0 parents  commit bb89d9d

File tree

567 files changed

+2115
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

567 files changed

+2115
-0
lines changed

‎.github/hubmirror/__init__.py‎

Whitespace-only changes.

‎.github/hubmirror/hub.py‎

Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,228 @@
1+
import time
2+
import functools
3+
import json
4+
5+
import requests
6+
7+
8+
class Hub(object):
9+
def __init__(
10+
self, src, dst, dst_token, account_type="user",
11+
clone_style="https",
12+
src_account_type=None,
13+
dst_account_type=None,
14+
api_timeout=60,
15+
gitea_url=None,
16+
):
17+
self.api_timeout = api_timeout
18+
self.account_type = account_type
19+
self.src_account_type = src_account_type or account_type
20+
self.dst_account_type = dst_account_type or account_type
21+
self.src_type, self.src_account = src.split('/')
22+
self.dst_type, self.dst_account = dst.split('/')
23+
self._validate_account_type(
24+
self.src_type, self.src_account_type, 'source'
25+
)
26+
self._validate_account_type(
27+
self.dst_type, self.dst_account_type, 'destination'
28+
)
29+
self.dst_token = dst_token
30+
self.session = requests.Session()
31+
if self.dst_type == "gitea":
32+
self.dst_base = f'{gitea_url}/api/v1'
33+
elif self.dst_type == "gitee":
34+
self.dst_base = 'https://gitee.com/api/v5'
35+
elif self.dst_type == "github":
36+
self.dst_base = 'https://api.github.com'
37+
elif self.dst_type == "gitlab":
38+
self.dst_base = 'https://gitlab.com/api/v4'
39+
40+
prefix = "https://" if clone_style == 'https' else 'git@'
41+
suffix = "/" if clone_style == 'https' else ':'
42+
if self.src_type == "gitea":
43+
self.src_base = f'{gitea_url}/api/v1'
44+
self.src_repo_base = prefix + gitea_url.replace("http://", "").replace("https://", "") + suffix
45+
elif self.src_type == "gitee":
46+
self.src_base = 'https://gitee.com/api/v5'
47+
self.src_repo_base = prefix + 'gitee.com' + suffix
48+
elif self.src_type == "github":
49+
self.src_base = 'https://api.github.com'
50+
self.src_repo_base = prefix + 'github.com' + suffix
51+
print(f"DEBUG self.src_repo_base if gitea: {prefix + gitea_url.replace("http://", "").replace("https://", "") + suffix}")
52+
elif self.src_type == "gitlab":
53+
self.src_base = 'https://gitlab.com/api/v4'
54+
self.src_repo_base = prefix + 'gitlab.com' + suffix
55+
self.src_repo_base = self.src_repo_base + self.src_account
56+
# TODO: toekn push support
57+
if self.dst_type == "gitea":
58+
#[email protected]:minlearn /inst.git
59+
self.dst_repo_base = "gitea@" + gitea_url.replace("http://", "").replace("https://", "").replace("/gitea", "") + ":" + self.dst_account
60+
print(f"DEBUG self.dst_repo_base if gitea: {self.dst_repo_base}")
61+
else:
62+
prefix = "git@" + self.dst_type + ".com:"
63+
self.dst_repo_base = prefix + self.dst_account
64+
65+
def _validate_account_type(self, platform_type, account_type, role):
66+
if platform_type not in ("gitlab", "github", "gitee", "gitea"):
67+
raise ValueError(
68+
f"Unsupported platform_type '{platform_type}' for {role}."
69+
)
70+
# gitea ---> user or org
71+
if platform_type == "gitea":
72+
if account_type not in ("user", "org"):
73+
raise ValueError(
74+
f"For {platform_type}, {role} account_type must be "
75+
"either 'user' or 'org'."
76+
)
77+
# gitlab ---> user or group
78+
elif platform_type == "gitlab":
79+
if account_type not in ("user", "group"):
80+
raise ValueError(
81+
f"For {platform_type}, {role} account_type must be "
82+
"either 'user' or 'group'."
83+
)
84+
# github/gitee ---> user or org
85+
elif platform_type in ("github", "gitee"):
86+
if account_type not in ("user", "org"):
87+
raise ValueError(
88+
f"For {platform_type}, {role} account_type must be"
89+
"either 'user' or 'org'."
90+
)
91+
92+
def has_dst_repo(self, repo_name):
93+
# gitlab ---> projects, github/gitee ---> repos
94+
repo_field = "projects" if self.dst_type == "gitlab" else "repos"
95+
url = '/'.join(
96+
[
97+
self.dst_base, self.dst_account_type+'s', self.dst_account,
98+
repo_field,
99+
]
100+
)
101+
repo_names = self._get_all_repo_names(url)
102+
if not repo_names:
103+
print("Warning: destination repos is []")
104+
return False
105+
return repo_name in repo_names
106+
107+
def create_dst_repo(self, repo_name):
108+
result = None
109+
# gitlab ---> projects, github/gitee ---> repos
110+
repo_field = "projects" if self.dst_type == "gitlab" else "repos"
111+
if self.dst_type == "gitlab":
112+
url = f"{self.dst_base}/{repo_field}"
113+
headers = {'PRIVATE-TOKEN': self.dst_token}
114+
data = {'name': repo_name, 'visibility': 'public'}
115+
# If creating under a group, add namespace_id
116+
if self.dst_account_type == "group":
117+
group_id = self._get_gitlab_group_id(self.dst_account)
118+
data['namespace_id'] = group_id
119+
else:
120+
suffix = f"user/{repo_field}"
121+
if self.dst_account_type == "org":
122+
suffix = f"orgs/{self.dst_account}/{repo_field}"
123+
url = '/'.join(
124+
[self.dst_base, suffix]
125+
)
126+
if self.dst_type == 'gitee':
127+
data = {'name': repo_name}
128+
elif self.dst_type == 'github':
129+
data = json.dumps({'name': repo_name})
130+
if not self.has_dst_repo(repo_name):
131+
print(repo_name + " doesn't exist, create it...")
132+
if self.dst_type == "gitea":
133+
response = self.session.get(
134+
f"{self.dst_base}/repos/{self.dst_account}/{repo_name}",
135+
headers={'Authorization': f'token {self.dst_token}'},
136+
timeout=self.api_timeout
137+
)
138+
print(f"DEBUG self.create_dst_repo if gitea: {self.dst_base}/repos/{self.dst_account}/{repo_name}")
139+
return response.status_code == 200
140+
elif self.dst_type == "github":
141+
response = self.session.post(
142+
url,
143+
data=data,
144+
headers={'Authorization': 'token ' + self.dst_token},
145+
timeout=self.api_timeout
146+
)
147+
result = response.status_code == 201
148+
if result:
149+
print("Destination repo creating accepted.")
150+
else:
151+
print("Destination repo creating failed: " + response.text)
152+
elif self.dst_type == "gitee":
153+
response = requests.post(
154+
url,
155+
headers={'Content-Type': 'application/json;charset=UTF-8'},
156+
params={"name": repo_name, "access_token": self.dst_token},
157+
timeout=self.api_timeout
158+
)
159+
result = response.status_code == 201
160+
if result:
161+
print("Destination repo creating accepted.")
162+
else:
163+
print("Destination repo creating failed: " + response.text)
164+
elif self.dst_type == "gitlab":
165+
response = self.session.post(
166+
url,
167+
data=data,
168+
headers=headers,
169+
timeout=self.api_timeout
170+
)
171+
result = response.status_code == 201
172+
if result:
173+
print("Destination repo creating accepted.")
174+
else:
175+
print("Destination repo creating failed: " + response.text)
176+
else:
177+
print(repo_name + " repo exist, skip creating...")
178+
# TODO(snowyu): Cleanup 2s sleep
179+
if result:
180+
time.sleep(2)
181+
return result
182+
183+
def dynamic_list(self):
184+
# gitlab ---> projects, github/gitee ---> repos
185+
repo_field = "projects" if self.src_type == "gitlab" else "repos"
186+
url = '/'.join(
187+
[
188+
self.src_base, self.src_account_type + 's', self.src_account,
189+
repo_field,
190+
]
191+
)
192+
return self._get_all_repo_names(url)
193+
194+
@functools.lru_cache
195+
def _get_all_repo_names(self, url, page=1):
196+
per_page = 60
197+
api = url + f"?page={page}&per_page=" + str(per_page)
198+
# TODO: src_token support
199+
response = self.session.get(api, timeout=self.api_timeout)
200+
all_items = []
201+
if response.status_code != 200:
202+
print("Repo getting failed: " + response.text)
203+
return all_items
204+
items = response.json()
205+
if items:
206+
names = [i['name'] for i in items]
207+
return names + self._get_all_repo_names(url, page=page+1)
208+
return all_items
209+
210+
def _get_gitlab_group_id(self, group_name):
211+
"""Helper method to get GitLab group ID"""
212+
url = f"{self.dst_base}/groups"
213+
headers = {'PRIVATE-TOKEN': self.dst_token}
214+
response = self.session.get(
215+
url,
216+
headers=headers,
217+
timeout=self.api_timeout
218+
)
219+
if response.status_code == 200:
220+
groups = response.json()
221+
for group in groups:
222+
if group['path'] == group_name:
223+
return group['id']
224+
print(f"Failed to find group ID for '{group_name}'.")
225+
else:
226+
print("Failed to get groups list.")
227+
print(f"Error message: {response.text}")
228+
return None

‎.github/hubmirror/hubmirror.py‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
import argparse
2+
import sys
3+
4+
from utils import str2bool, str2list, str2map
5+
from hub import Hub
6+
from mirror import Mirror
7+
8+
9+
class HubMirror(object):
10+
def __init__(self):
11+
self.parser = self._create_parser()
12+
self.args = self.parser.parse_args()
13+
self.white_list = str2list(self.args.white_list)
14+
self.black_list = str2list(self.args.black_list)
15+
self.static_list = str2list(self.args.static_list)
16+
self.mappings = str2map(self.args.mappings)
17+
18+
def _create_parser(self):
19+
parser = argparse.ArgumentParser(
20+
description="Mirror the organization repos between hub (gitea/github/gitee/gitLab)."
21+
)
22+
# Add arguments with default values and descriptions
23+
parser.add_argument("--dst-key", type=str, required=True, help="The private SSH key used to push code in the destination hub.")
24+
parser.add_argument("--dst-token", type=str, required=True, help="The app token used to create repos in the destination hub.")
25+
parser.add_argument("--dst", type=str, required=True, help="Destination name. Such as `gitee/kunpengcompute`.")
26+
parser.add_argument("--gitea-url", type=str, default="", help="The base URL for your Gitea instance, e.g., `https://your.gitea.instance`.")
27+
parser.add_argument("--src", type=str, required=True, help="Source name. Such as `github/kunpengcompute`.")
28+
parser.add_argument("--account-type", type=str, default="user", help="The account type. Such as org, user, group.")
29+
parser.add_argument("--src-account-type", type=str, default="", help="The src account type. Such as org, user, group.")
30+
parser.add_argument("--dst-account-type", type=str, default="", help="The dst account type. Such as org, user, group.")
31+
parser.add_argument("--clone-style", type=str, default="https", help="The git clone style, https or ssh.")
32+
parser.add_argument("--cache-path", type=str, default="/github/workspace/hub-mirror-cache", help="The path to cache the source repos code.")
33+
parser.add_argument("--black-list", type=str, default="", help="High priority, the blacklist of mirror repos, separated by commas.")
34+
parser.add_argument("--white-list", type=str, default="", help="Low priority, the whitelist of mirror repos, separated by commas.")
35+
parser.add_argument("--static-list", type=str, default="", help="Only mirror repos in the static list, separated by commas.")
36+
parser.add_argument("--force-update", type=str2bool, default=False, help="Force to update the destination repo, use '-f' flag for 'git push'.")
37+
parser.add_argument("--debug", type=str2bool, default=False, help="Enable the debug flag to show detailed log.")
38+
parser.add_argument("--timeout", type=str, default="30m", help="Set the timeout for every git command, e.g., '600'=600s, '30m'=30 mins.")
39+
parser.add_argument("--api-timeout", type=int, default=60, help="Set the timeout for API requests (in seconds).")
40+
parser.add_argument("--mappings", type=str, default="", help="The source repos mappings, e.g., 'A=>B, C=>CC'. Source repo name would be mapped accordingly.")
41+
parser.add_argument("--lfs", type=str2bool, default=False, help="Enable Git LFS support.")
42+
return parser
43+
44+
def test_black_white_list(self, repo):
45+
if repo in self.black_list:
46+
print(f"Skip, {repo} in black list: {self.black_list}")
47+
return False
48+
49+
if self.white_list and repo not in self.white_list:
50+
print(f"Skip, {repo} not in white list: {self.white_list}")
51+
return False
52+
53+
return True
54+
55+
def run(self):
56+
hub = Hub(
57+
self.args.src,
58+
self.args.dst,
59+
self.args.dst_token,
60+
account_type=self.args.account_type,
61+
clone_style=self.args.clone_style,
62+
src_account_type=self.args.src_account_type,
63+
dst_account_type=self.args.dst_account_type,
64+
api_timeout=int(self.args.api_timeout),
65+
gitea_url=self.args.gitea_url
66+
)
67+
src_type, src_account = self.args.src.split('/')
68+
69+
# Using static list when static_list is set
70+
repos = self.static_list
71+
src_repos = repos if repos else hub.dynamic_list()
72+
73+
total, success, skip = len(src_repos), 0, 0
74+
failed_list = []
75+
for src_repo in src_repos:
76+
# Set dst_repo to src_repo mapping or src_repo directly
77+
dst_repo = self.mappings.get(src_repo, src_repo)
78+
print(f"Map {src_repo} to {dst_repo}")
79+
if self.test_black_white_list(src_repo):
80+
print(f"Backup {src_repo}")
81+
try:
82+
mirror = Mirror(
83+
hub, src_repo, dst_repo,
84+
cache=self.args.cache_path,
85+
timeout=self.args.timeout,
86+
force_update=self.args.force_update,
87+
lfs=(
88+
self.args.lfs if hasattr(self.args, "lfs")
89+
else False
90+
)
91+
)
92+
mirror.download()
93+
mirror.create()
94+
mirror.push()
95+
success += 1
96+
except Exception as e:
97+
print(e)
98+
failed_list.append(src_repo)
99+
else:
100+
skip += 1
101+
failed = total - success - skip
102+
res = (total, skip, success, failed)
103+
print(f"Total: {total}, skip: {skip}, successed: {success}, failed: {failed}.")
104+
print(f"Failed: {failed_list}")
105+
if failed_list:
106+
sys.exit(1)
107+
108+
109+
if __name__ == '__main__':
110+
mirror = HubMirror()
111+
mirror.run()

0 commit comments

Comments
 (0)