Upload only .db files from the main directory

Update taginfo script and readme

Update taginfo

Path to taginfo to sabe db files

Install cron  in taginfo

Fetch data if requiere - taginfo

Update config

Author: Rub21

envs/.env.taginfo.example

@@ -14,3 +14,6 @@ TAGINFO_PROJECT_REPO=https://github.com/OpenHistoricalMap/taginfo-projects.git
 DOWNLOAD_DB='languages wiki'
 CREATE_DB='db projects chronology'
 TAGINFO_DB_BASE_URL=https://planet.openhistoricalmap.org.s3.amazonaws.com/taginfo
+AWS_S3_BUCKET=planet-staging.openhistoricalmap.org
+ENVIRONMENT=staging
+FETCH_DB_FILES=true

images/taginfo/Dockerfile

@@ -25,6 +25,7 @@ RUN apt-get update && apt-get install -y \
     jq \
     python3-pip \
     wget \
+    cron \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 

images/taginfo/README.md

@@ -1,41 +1,43 @@
 # OSM-Seed taginfo
 
-We build a docker container for taginfo software, the container will start the web service and also process required files to create databases.
+Docker container for taginfo that runs the web service and processes PBF files to create databases.
 
 ## Environment Variables
 
-All environment variables are located at [`.env.taginfo.example`](./../../envs/.env.taginfo.example), make a copy and name it as `.env.tagninfo` to use in osm-seed.
+Copy [`.env.taginfo.example`](./../../envs/.env.taginfo.example) to `.env.taginfo` and configure:
 
-- `URL_PLANET_FILE_STATE`: Url to the state file, that contains the URL for the latest planet PBF file. e.g [`state.txt`](https://planet.openhistoricalmap.org.s3.amazonaws.com/planet/state.txt), This is no required in case you set the `URL_PLANET_FILE` env var
+### Planet Files
+- `URL_PLANET_FILE_STATE`: URL to state file with latest planet PBF URL (optional if `URL_PLANET_FILE` is set)
+- `URL_HISTORY_PLANET_FILE_STATE`: URL to state file with latest history PBF URL (optional if `URL_HISTORY_PLANET_FILE` is set)
+- `URL_PLANET_FILE`: Direct URL to planet PBF file
+- `URL_HISTORY_PLANET_FILE`: Direct URL to history PBF file
 
-- `URL_HISTORY_PLANET_FILE_STATE`: Url to the full history state file, that contains the URL for the latest full history planet PBF file. e.g [`state.txt`](https://planet.openhistoricalmap.org.s3.amazonaws.com/planet/full-history/state.txt), This is no required in case you set the `URL_HISTORY_PLANET_FILE` env var
+### Database Configuration
+- `TAGINFO_DB_BASE_URL`: Base URL to download SQLite database files. Downloads: projects-cache.db, selection.db, taginfo-chronology.db, taginfo-db.db, taginfo-history.db, taginfo-languages.db, taginfo-master.db, taginfo-projects.db, taginfo-wiki.db, taginfo-wikidata.db
+  - Example: `https://planet.openhistoricalmap.org.s3.amazonaws.com/taginfo`
 
-- `URL_PLANET_FILE`: URL for the latest planet PBF file.
-- `URL_HISTORY_PLANET_FILE`: URL for the latest full history planet PBF file.
-- `TIME_UPDATE_INTERVAL` Interval time to update the databases, e.g: `50m` = every 50 minutes, `20h` = every 20 hours , `5d` = every 5 days
+- `DOWNLOAD_DB`: Which databases to download (e.g., `languages wiki` or `languages wiki projects chronology`)
 
-The following env vars are required in the instance to update the values at: https://github.com/taginfo/taginfo/blob/master/taginfo-config-example.json
+- `CREATE_DB`: Which databases to create from PBF files (e.g., `db projects` or `db projects chronology`)
+  - `db` requires `URL_PLANET_FILE` or `URL_PLANET_FILE_STATE`
+  - `projects` requires `TAGINFO_PROJECT_REPO`
+  - `chronology` requires `URL_PLANET_FILE` or `URL_HISTORY_PLANET_FILE`
 
-- `OVERWRITE_CONFIG_URL`: config file with the values to update
+### Other
+- `TAGINFO_PROJECT_REPO`: Repository URL for taginfo projects (default: https://github.com/taginfo/taginfo-projects.git)
+- `OVERWRITE_CONFIG_URL`: URL to custom taginfo config JSON file
+- `INTERVAL_DOWNLOAD_DATA`: Interval to sync databases (e.g., `3600` for 1 hour, `7d` for 7 days)
 
-- `DOWNLOAD_DB`: Taginfo instances need 7 Sqlite databases to start up the web service, all of them can be downloaded from https://taginfo.openstreetmap.org/download. Or if you can download only some of them you can pass herec. e.g DOWNLOAD_DB=`languages wiki`, or DOWNLOAD_DB=`languages wiki projects chronology`.
-
-- `CREATE_DB`: If you want process you of data using the PBF files, you can pass the values. eg. CREATE_DB=`db projects` or CREATE_DB=`db projects chronology`.
-  Note: 
-  - Value `db` require to pass `URL_PLANET_FILE` or `URL_PLANET_FILE_STATE` 
-  - Value `projects` require to pass `TAGINFO_PROJECT_REPO` 
-  - Value `chronology` require to pass `URL_PLANET_FILE` or `URL_HISTORY_PLANET_FILE`
-
-#### Running taginfo container
+## Running
 
 ```sh
-    # Docker compose
-    docker-compose run taginfo
-
-    # Docker
-    docker run \
-    --env-file ./envs/.env.taginfo \
-    -v ${PWD}/data/taginfo-data:/apps/data/ \
-    --network osm-seed_default \
-    -it osmseed-taginfo:v1
-```
+# Docker compose
+docker-compose run taginfo
+
+# Docker
+docker run \
+  --env-file ./envs/.env.taginfo \
+  -v ${PWD}/data/taginfo-data:/usr/src/app/data \
+  --network osm-seed_default \
+  -it osmseed-taginfo:v1
+```

images/taginfo/start.sh

@@ -46,8 +46,12 @@ process_data() {
     # wikidata/update.sh $DATADIR
     chronology/update.sh $DATADIR
     ./update_all.sh $DATADIR
-    # Move database files from subdirectories to main data directory
-    aws s3 sync $DATADIR/ s3://$AWS_S3_BUCKET/taginfo/$ENVIRONMENT/  --exclude "*" --include "*.db"
+    mv $DATADIR/*.db $DATADIR/
+    mv $DATADIR/*/*.db $DATADIR/
+    # if AWS_S3_BUCKET is set upload data
+    if ! aws s3 ls "s3://$AWS_S3_BUCKET/taginfo" 2>&1 | grep -q 'An error occurred'; then
+        aws s3 sync $DATADIR/ s3://$AWS_S3_BUCKET/taginfo/  --exclude "*" --include "*.db"
+    fi
 }
 
 # Compress files to download
@@ -105,22 +109,26 @@ download_db_files() {
 
 sync_latest_db_version() {
     while true; do
-        sleep "$INTERVAL_DOWNLOAD_DATA"
         download_db_files "$TAGINFO_DB_BASE_URL"
+        sleep "$INTERVAL_DOWNLOAD_DATA"
     done
 }
 
 start_web() {
     echo "Start...Taginfo web service"
-    download_db_files "$TAGINFO_DB_BASE_URL"
-    cd $WORKDIR/taginfo/web && ./taginfo.rb & sync_latest_db_version
+    cd $WORKDIR/taginfo/web && ./taginfo.rb
 }
 
 ACTION=$1
 # Overwrite the config file
 [[ ! -z ${OVERWRITE_CONFIG_URL} ]] && wget $OVERWRITE_CONFIG_URL -O /usr/src/app/taginfo-config.json
 updates_source_code
 if [ "$ACTION" = "web" ]; then
+    # Start sync in background if enabled
+    if [ "${FETCH_DB_FILES:-true}" = "true" ] && [ ! -z "$TAGINFO_DB_BASE_URL" ]; then
+        sync_latest_db_version &
+    fi
+    # Start web server in foreground (so the loop can detect if it fails)
     start_web
     elif [ "$ACTION" = "data" ]; then
     process_data