Skip to content

WIP: Generic arweave capability (NOT TO BE MERGED) #38

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

WIP: Generic arweave capability (NOT TO BE MERGED) #38

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ name = "arweave-api"
#
dynamic = ["version"]

description = "Arkly Arweave API"
description = "NOT Arkly Arweave API"
readme = "README.md"

# Supported python versions. Optional, but helpful.
Expand Down
35 changes: 9 additions & 26 deletions src/arweave_api/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,15 @@

try:
import primary_functions
from middleware import _update_db
from models import Tags
from version import get_version
except ModuleNotFoundError:
try:
from src.arweave_api import primary_functions
from src.arweave_api.middleware import _update_db
from src.arweave_api.models import Tags
from src.arweave_api.version import get_version
except ModuleNotFoundError:
from arweave_api import primary_functions
from arweave_api.middleware import _update_db
from arweave_api.models import Tags
from arweave_api.version import get_version

Expand All @@ -33,7 +30,7 @@
TAG_ARWEAVE: Final[str] = "arweave"
TAG_ARWEAVE_WALLET: Final[str] = "arweave wallet"
TAG_ARWEAVE_SEARCH: Final[str] = "arweave search"
TAG_ARKLY: Final[str] = "arkly"
TAG_ARKLY: Final[str] = "packaging"
TAG_MAINTAIN: Final[str] = "maintenance"

# Metadata for each of the tags in the OpenAPI specification. To order
Expand All @@ -53,12 +50,12 @@
},
{
"name": TAG_ARKLY,
"description": "Arkly functions on-top of Arweave",
"description": " functions on-top of Arweave",
},
]

app = FastAPI(
title="api.arkly.io",
title="arweave API",
description=API_DESCRIPTION,
version=get_version(),
contact={
Expand All @@ -77,14 +74,6 @@
)


@app.middleware("http")
async def update_db(request: Request, call_next):
"""Middleware used to identify which endpoint is being used so that
the database can be updated effectively.
"""
return await _update_db(request, call_next)


@app.get("/", include_in_schema=False)
def redirect_root_to_docs():
"""Redirect a user calling the API root '/' to the API
Expand Down Expand Up @@ -198,25 +187,19 @@ async def get_transactions_by_tag_pair(name: str, value: str):
@app.post("/create_transaction/", tags=[TAG_ARKLY])
async def create_transaction(
wallet: UploadFile,
package_file_name: str,
files: List[UploadFile] = File(...),
file: list[UploadFile] = File(...),
mime_type: str = None,
tags: Tags | None = None,
):
"""Create an Arkly package and Arweave transaction."""
return await primary_functions._create_transaction(
wallet, files, package_file_name, tags
wallet=wallet,
files=file,
mime_type=mime_type,
tags=tags,
)


@app.get("/validate_arkly_bag/", tags=[TAG_ARKLY])
async def validate_bag(transaction_id: str, response: Response):
"""Given an Arweave transaction ID, Validate an Arkly link as a bag.

Example Tx: `rYa3ILXqWi_V52xPoG70y2EupPsTtu4MsMmz6DI4fy4`
"""
return await primary_functions._validate_bag(transaction_id, response)


@app.get("/get_version/", tags=[TAG_MAINTAIN])
async def get_version_info():
"""Return API version information to the caller."""
Expand Down
38 changes: 0 additions & 38 deletions src/arweave_api/middleware.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,41 +11,3 @@
from fastapi import Request

logger = logging.getLogger(__name__)


async def _update_db(request: Request, call_next: Callable):
"""Update the database by one per endpoint called."""
path = str(request.scope["path"])
endpoints = [
"/docs",
"/check_balance/",
"/check_last_transaction/",
"/check_transaction_status",
"/create_transaction/",
"/estimate_traansaction_cost",
"/fetch_upload/",
"/validate_arweave_bag/",
]
# Update database endpoint_calls by 1
if path in endpoints:
if path == "/docs":
path = "root"
else:
# Remove first and last character
path = path[1:-1]
try:
connection = psycopg2.connect(
user="arkly", host="/var/run/postgresql/", database="arkly", port=5432
)
cursor = connection.cursor()
update_endpoint_count = """UPDATE endpoint_calls
SET {update_db_endpoint} = {update_db_endpoint} + 1""".format(
update_db_endpoint=path
)
cursor.execute(update_endpoint_count)
connection.commit()
cursor.close()
except psycopg2.DatabaseError as error:
logger.warning("Postgres may not be configured correctly: %s", error)
response = await call_next(request)
return response
134 changes: 21 additions & 113 deletions src/arweave_api/primary_functions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
from typing import Final, List

import arweave
import bagit
import humanize
import requests
from arweave.arweave_lib import Transaction, arql
Expand Down Expand Up @@ -322,55 +321,10 @@ async def _fetch_upload(transaction_id: str) -> FileResponse:
raise HTTPException from err


async def bag_files(path: Path, tag_list=None) -> None:
"""Use python Bagit to bag the files for Arkly-Arweave."""
if not tag_list:
bagit.make_bag(path, {PACKAGING_AGENT_STRING: ARKLY_AGENT})
return
bag_info = {}
for tag in tag_list:
bag_info[f"{tag.name}".replace(" ", "-")] = tag.value
bag_info[PACKAGING_AGENT_STRING] = ARKLY_AGENT
logger.info("writing package with bag-info: %d", bag_info)
bagit.make_bag(path, bag_info)
return


async def _package_content(
files: List[UploadFile] = File(...), package_name: str = None, tag_list: list = None
) -> dict:
"""Package the files submitted to the create_transaction endpoint."""
# Create a folder for the user's wallet.
tmp_dir = tempfile.mkdtemp()
file_path = Path(tmp_dir, package_name)
file_path.mkdir()

logger.info("Location to write object to: %s", file_path)

for file in files:
read_file = await file.read()
output_file = Path(file_path, file.filename)
output_file.write_bytes(read_file)

# Bag these files.
await bag_files(file_path, tag_list)

# Create compressed .tar.gz file
tar_file_name = file_path.with_suffix(".tar.gz")
with tarfile.open(tar_file_name, "w:gz") as tar:
tar.add(file_path, arcname=os.path.basename(file_path))

version_api: Final[str] = "v0"
tar_file_name = tar_file_name.rename(
f"{tar_file_name}".replace(".tar.gz", f"_{version_api}.tar.gz")
)
return tar_file_name


async def _create_transaction(
wallet: UploadFile,
files: List[UploadFile] = File(...),
package_file_name: str = None,
files: list[UploadFile] = File(...),
mime_type: str = None,
tags: Tags = None,
) -> dict:
"""Create an Arkly package and Arweave transaction.
Expand Down Expand Up @@ -398,21 +352,29 @@ async def _create_transaction(
except AttributeError:
logger.info("no user-defined tags provided by caller")

# Create a package from files array. Package content will create
# this in a secure temporary directory.
tar_file_name = await _package_content(files, package_file_name, tag_list)

logger.info("Adding version to package: %s", tar_file_name)
logger.info("New path exists: %s", tar_file_name.is_file())
logger.info("Wallet balance before upload: %s", wallet.balance)

with open(tar_file_name, "rb", buffering=0) as file_handler:
tmp_dir = tempfile.mkdtemp()
file_path = Path(tmp_dir, "upload_path")
file_path.mkdir()

logger.info("Location to write object to: %s", file_path)

output_file = None
for file in files:
read_file = await file.read()
output_file = Path(file_path, file.filename)
output_file.write_bytes(read_file)
break

logger.info("attempting to upload: %s", output_file)

with open(output_file, "rb", buffering=0) as file_handler:
new_transaction = Transaction(
wallet, file_handler=file_handler, file_path=tar_file_name
wallet, file_handler=file_handler, file_path=output_file,
)
# Default tags for the tar/gzip file that we create.
new_transaction.add_tag("Content-Type", "application/gzip")

new_transaction.add_tag("Content-Type", mime_type)
for tag in tag_list:
logger.info("Adding tag: %s: %s", tag.name, tag.value)
new_transaction.add_tag(tag.name, tag.value)
Expand Down Expand Up @@ -445,60 +407,6 @@ def _get_arweave_urls_from_tx(transaction_id: str) -> dict:
)


async def _validate_bag(transaction_id: str, response: Response) -> dict:
"""Given an Arweave transaction ID, Validate an Arkly link as a bag."""

# Setup retrieval of the data from the given transaction.
transaction_url, arweave_url = _get_arweave_urls_from_tx(transaction_id)
arweave_response = requests.get(arweave_url, allow_redirects=True)

# Create temp file to extract the contents from Arweave to.
tmp_file_handle, tmp_file_path = tempfile.mkstemp()
with open(tmp_file_handle, "wb") as write_tar_gz:
write_tar_gz.write(arweave_response.content)

tmp_dir = tempfile.mkdtemp()
try:
arkly_gzip = tarfile.open(tmp_file_path)
arkly_gzip.extractall(tmp_dir)
except tarfile.ReadError:
response.status_code = status.HTTP_422_UNPROCESSABLE_ENTITY
return {
"transaction_url": transaction_url,
"file_url": arweave_url,
"valid": "UNKNOWN",
}

try:
bag_name = os.listdir(tmp_dir)[0]
bag_file = Path(tmp_dir) / bag_name
except IndexError:
response.status_code = status.HTTP_404_NOT_FOUND
return {
"transaction_url": transaction_url,
"file_url": arweave_url,
"valid": "UNKNOWN",
}

# Create bag object and validate, and return information from it.
try:
arkly_bag = bagit.Bag(str(bag_file))
return {
"transaction_url": transaction_url,
"file_url": arweave_url,
"valid": f"{arkly_bag.validate()}",
"bag_info": arkly_bag.info,
"bag_name": bag_name,
}
except bagit.BagError:
response.status_code = status.HTTP_422_UNPROCESSABLE_ENTITY
return {
"transaction_url": transaction_url,
"file_url": arweave_url,
"valid": "UNKNOWN",
}


async def _all_transactions(wallet_addr: str):
"""Retrieve all transactions from a given wallet and return a human
friendly link to enable users to view the transaction.
Expand Down Expand Up @@ -537,4 +445,4 @@ async def _retrieve_by_tag_pair(name: str, value: str) -> dict:

async def _get_version_info() -> dict:
"""Return information about the versions used by this API."""
return {"api": get_version(), "agent": ARKLY_AGENT, "bagit": bagit.VERSION}
return {"api": get_version()}