Skip to content

add check if repo exists for dataset uploading #7565

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed

add check if repo exists for dataset uploading #7565

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3772dd0
add check if repo exists
Samoed May 9, 2025
58c9493
Merge branch 'main' into fix_datasets_repo_create
lhoestq May 19, 2025
43e3438
fix import
Samoed May 19, 2025
24fa715
Merge branch 'main' into fix_datasets_repo_create
Samoed Jun 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 12 additions & 6 deletions src/datasets/arrow_dataset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@
DatasetCard,
DatasetCardData,
HfApi,
constants,
)
from huggingface_hub.hf_api import RepoFile
from multiprocess import Pool
Expand Down Expand Up @@ -5607,14 +5608,19 @@ def push_to_hub(

api = HfApi(endpoint=config.HF_ENDPOINT, token=token)

repo_url = api.create_repo(
if not api.repo_exists(
repo_id,
token=token,
repo_type="dataset",
private=private,
exist_ok=True,
)
repo_id = repo_url.repo_id
repo_type=constants.REPO_TYPE_DATASET,
):
repo_url = api.create_repo(
repo_id,
token=token,
repo_type=constants.REPO_TYPE_DATASET,
private=private,
exist_ok=True,
)
repo_id = repo_url.repo_id

if revision is not None and not revision.startswith("refs/pr/"):
# We do not call create_branch for a PR reference: 400 Bad Request
Expand Down
18 changes: 12 additions & 6 deletions src/datasets/dataset_dict.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
DatasetCard,
DatasetCardData,
HfApi,
constants,
)
from huggingface_hub.hf_api import RepoFile

Expand Down Expand Up @@ -1720,14 +1721,19 @@ def push_to_hub(

api = HfApi(endpoint=config.HF_ENDPOINT, token=token)

repo_url = api.create_repo(
if not api.repo_exists(
repo_id,
token=token,
repo_type="dataset",
private=private,
exist_ok=True,
)
repo_id = repo_url.repo_id
repo_type=constants.REPO_TYPE_DATASET,
):
repo_url = api.create_repo(
repo_id,
token=token,
repo_type=constants.REPO_TYPE_DATASET,
private=private,
exist_ok=True,
)
repo_id = repo_url.repo_id

if revision is not None and not revision.startswith("refs/pr/"):
# We do not call create_branch for a PR reference: 400 Bad Request
Expand Down