chore: update configuration for Robotoff Moji migration

- remove legacy tf_models
- remove container-deploy-ml.yml
- update configuration for container deploy
- remove Robotoff PostgreSQL backup (which was OVH1-specific)

.dockerignore

@@ -5,7 +5,6 @@ data
 datasets
 local_models
 models
-tf_models
 .git
 .idea
 weights

.github/workflows/container-deploy.yml

@@ -18,35 +18,42 @@ jobs:
     environment: ${{ matrix.env }}
     concurrency: ${{ matrix.env }}
     steps:
-    - name: Set common variables
-      run: |
-        echo "SSH_PROXY_HOST=ovh1.openfoodfacts.org" >> $GITHUB_ENV
-        echo "SSH_USERNAME=off" >> $GITHUB_ENV
     - name: Set various variable for staging deployment
       if: matrix.env == 'robotoff-net'
       run: |
+        echo "SSH_PROXY_HOST=ovh1.openfoodfacts.org" >> $GITHUB_ENV
+        echo "SSH_USERNAME=off" >> $GITHUB_ENV
         echo "SSH_HOST=10.1.0.200" >> $GITHUB_ENV
         echo "ROBOTOFF_INSTANCE=dev" >> $GITHUB_ENV
         echo "ROBOTOFF_TLD=net" >> $GITHUB_ENV
         echo "MONGO_URI=mongodb://10.1.0.200:27017" >> $GITHUB_ENV
         # In staging, we use the Redis of shared services to get Product Update
         echo "REDIS_UPDATE_HOST=redis.po_webnet" >> $GITHUB_ENV
         echo "INFLUXDB_HOST=10.1.0.200" >> $GITHUB_ENV
+        echo "INFLUXDB_PORT=8087" >> $GITHUB_ENV
         echo "IMAGE_MODERATION_SERVICE_URL=https://nutripatrol.openfoodfacts.net/api/v1/flags" >> $GITHUB_ENV
         echo "CROP_ALLOWED_DOMAINS=static.openfoodfacts.net,static.openfoodfacts.org,openfoodfacts-images.s3.eu-west-3.amazonaws.com,images.openfoodfacts.net,images.openfoodfacts.org" >> $GITHUB_ENV
+        echo "ROBOTOFF_POSTGRES_SHARED_BUFFERS=8GB" >> $GITHUB_ENV
+        echo "ROBOTOFF_POSTGRES_WORK_MEM=1GB" >> $GITHUB_ENV
     - name: Set various variable for production deployment
       if: matrix.env == 'robotoff-org'
       run: |
-        echo "SSH_HOST=10.1.0.201" >> $GITHUB_ENV
+        # First connect on the SSH proxy, using ipv6 address
+        echo "SSH_PROXY_HOST=osm45.openstreetmap.fr" >> $GITHUB_ENV
+        echo "SSH_USERNAME=off" >> $GITHUB_ENV
+        echo "SSH_HOST=10.3.0.200" >> $GITHUB_ENV
         echo "ROBOTOFF_INSTANCE=prod" >> $GITHUB_ENV
         echo "ROBOTOFF_TLD=org" >> $GITHUB_ENV
         # use prod mongodb through stunnel
-        echo "MONGO_URI=mongodb://10.1.0.113:27017" >> $GITHUB_ENV
+        echo "MONGO_URI=mongodb://10.3.0.101:27017" >> $GITHUB_ENV
         # In prod, we use the Redis running in VM
-        echo "REDIS_UPDATE_HOST=10.1.0.113" >> $GITHUB_ENV
-        echo "INFLUXDB_HOST=10.1.0.201" >> $GITHUB_ENV
+        echo "REDIS_UPDATE_HOST=10.3.0.101" >> $GITHUB_ENV
+        echo "INFLUXDB_HOST=10.3.0.101" >> $GITHUB_ENV
+        echo "INFLUXDB_PORT=8086" >> $GITHUB_ENV
         echo "IMAGE_MODERATION_SERVICE_URL=https://nutripatrol.openfoodfacts.org/api/v1/flags" >> $GITHUB_ENV
         echo "CROP_ALLOWED_DOMAINS=static.openfoodfacts.org,openfoodfacts-images.s3.eu-west-3.amazonaws.com,images.openfoodfacts.org" >> $GITHUB_ENV
+        echo "ROBOTOFF_POSTGRES_SHARED_BUFFERS=16GB" >> $GITHUB_ENV
+        echo "ROBOTOFF_POSTGRES_WORK_MEM=2GB" >> $GITHUB_ENV
     - name: Wait for container build workflow
       uses: tomchv/[email protected]
       id: wait-build
@@ -114,7 +121,7 @@ jobs:
           echo "COMPOSE_HTTP_TIMEOUT=200" >> .env
           echo "COMPOSE_PROJECT_NAME=robotoff" >> .env
           echo "COMPOSE_PATH_SEPARATOR=;" >> .env
-          echo "COMPOSE_FILE=docker-compose.yml;docker/prod.yml;docker/monitor.yml" >> .env
+          echo "COMPOSE_FILE=docker-compose.yml;docker/prod.yml;docker/monitor.yml;docker/ml.yml" >> .env
           echo "TAG=sha-${{ github.sha }}" >> .env
           echo "RESTART_POLICY=always" >> .env
 
@@ -142,9 +149,12 @@ jobs:
           echo "ELASTIC_USER=elastic" >> .env
           echo "ELASTIC_PASSWORD=${{ secrets.ELASTIC_PASSWORD }}" >> .env
           echo "TRITON_HOST=triton" >> .env
+          echo "TRITON_EXPOSE_HTTP=8003" >> .env
+          echo "TRITON_MODELS_DIR=./models/triton" >> .env
           echo "FASTTEXT_HOST=fasttext" >> .env
+          echo "FASTTEXT_MODEL_DIR=./models" >> .env
           echo "INFLUXDB_HOST=${{ env.INFLUXDB_HOST }}" >> .env
-          echo "INFLUXDB_PORT=8087" >> .env
+          echo "INFLUXDB_PORT=${{ env.INFLUXDB_PORT }}" >> .env
           echo "INFLUXDB_BUCKET=off_metrics" >> .env
           echo "INFLUXDB_AUTH_TOKEN=${{ secrets.INFLUXDB_AUTH_TOKEN }}" >> .env
           echo "SLACK_TOKEN=${{ secrets.SLACK_TOKEN }}" >> .env
@@ -154,8 +164,8 @@ jobs:
           echo "CROP_ALLOWED_DOMAINS=${{ env.CROP_ALLOWED_DOMAINS }}" >> .env
 
           # PostgreSQL config
-          echo "ROBOTOFF_POSTGRES_SHARED_BUFFERS=8GB" >> .env
-          echo "ROBOTOFF_POSTGRES_WORK_MEM=1GB" >> .env
+          echo "ROBOTOFF_POSTGRES_SHARED_BUFFERS=${{ env.ROBOTOFF_POSTGRES_SHARED_BUFFERS }}" >> .env
+          echo "ROBOTOFF_POSTGRES_WORK_MEM=${{ env.ROBOTOFF_POSTGRES_WORK_MEM }}" >> .env
 
 
     - name: Create Docker volumes

.gitignore

@@ -27,7 +27,6 @@ images
 .venv/
 # pyenv file
 .python-version
-tf_models/*
 weights/
 
 # direnv

Makefile

@@ -18,10 +18,6 @@ DOCKER_COMPOSE=docker compose --env-file=${ENV_FILE}
 DOCKER_COMPOSE_TEST=COMPOSE_PROJECT_NAME=robotoff_test COMMON_NET_NAME=po_test docker compose --env-file=${ENV_FILE}
 ML_OBJECT_DETECTION_MODELS := tf-universal-logo-detector tf-nutrition-table tf-nutriscore
 
-# mount information for robotoff backup ZFS dataset
-NFS_VOLUMES_ADDRESS_OVH3 ?= 10.0.0.3
-NFS_VOLUMES_BACKUP_BASE_PATH ?= /rpool/backups/robotoff
-
 .DEFAULT_GOAL := dev
 # avoid target corresponding to file names, to depends on them
 .PHONY: *
@@ -239,12 +235,6 @@ create_external_volumes:
 	@echo "🥫 Creating external volumes (production only) …"
 	docker volume create robotoff_postgres-data
 	docker volume create robotoff_es-data
-# This is an NFS mount from robotoff backup ZFS dataset.
-# Two important notes:
-# - we use `nolock` as there shouldn't be any concurrent writes on the same file, and `soft` to prevent the docker container from freezing if the NFS
-#   connection is lost
-# - we cannot mount directly `${NFS_VOLUMES_BACKUP_BASE_PATH}`, we have to mount a subfolder (`backups`) to prevent permission issues
-	docker volume create --driver local --opt type=nfs --opt o=addr=${NFS_VOLUMES_ADDRESS_OVH3},nolock,soft,rw --opt device=:${NFS_VOLUMES_BACKUP_BASE_PATH}/backups ${COMPOSE_PROJECT_NAME}_backup
 
 
 create_external_networks:

docker-compose.yml

@@ -3,7 +3,6 @@ x-robotoff-base-volumes:
   - ./data:/opt/robotoff/data
   - ./cache:/opt/robotoff/cache
   - ./datasets:/opt/robotoff/datasets
-  - ./tf_models:/opt/robotoff/tf_models
   - ./models:/opt/robotoff/models
 
 x-robotoff-base:
@@ -126,7 +125,6 @@ services:
       - POSTGRES_DB
     volumes:
       - postgres-data:/var/lib/postgresql/data
-      - backup:/opt/robotoff-backups
       - ./scripts/backup_postgres.sh:/opt/backup_postgres.sh
     command: postgres -c shared_buffers=${ROBOTOFF_POSTGRES_SHARED_BUFFERS} -c work_mem=${ROBOTOFF_POSTGRES_WORK_MEM}
     mem_limit: 20g
@@ -168,8 +166,6 @@ services:
 volumes:
   postgres-data:
     name: ${COMPOSE_PROJECT_NAME:-robotoff}_postgres-data
-  backup:
-    name: ${COMPOSE_PROJECT_NAME:-robotoff}_backup
   es-data:
     name: ${COMPOSE_PROJECT_NAME:-robotoff}_es-data
   redis-data:

docker/prod.yml

@@ -1,8 +1,4 @@
 volumes:
-  backup:
-    external: true
-    name: ${COMPOSE_PROJECT_NAME:-robotoff}_backup
-
   postgres-data:
     external: true
     name: ${COMPOSE_PROJECT_NAME:-robotoff}_postgres-data