Shuffle

Resources

Shuffle documentation

Scripts

Updating Shuffle

Valid for both Manager & Worker servers.

shuffle-update.sh
cd /opt/Shuffle
docker-compose down
git pull
docker-compose pull
docker-compose up -d --remove-orphans
docker pull frikky/shuffle:app_sdk
docker pull ghcr.io/shuffle/shuffle-worker:latest

Configuration

Frontend / Backend / DB / "Manager"

Shuffle/docker-compose.yml
version: '3'
services:
  frontend:
    #build: ./frontend
    image: ghcr.io/shuffle/shuffle-frontend:latest
    container_name: shuffle-frontend
    hostname: shuffle-frontend
    ports:
      - "${FRONTEND_PORT}:80"
      - "${FRONTEND_PORT_HTTPS}:443"
    networks:
      - shuffle
    environment:
      - BACKEND_HOSTNAME=${BACKEND_HOSTNAME}
    restart: unless-stopped
    depends_on:
      - backend
  backend:
    #build: ./backend
    image: ghcr.io/shuffle/shuffle-backend:latest
    container_name: shuffle-backend
    hostname: ${BACKEND_HOSTNAME}
    # Here for debugging:
    ports:
      - "${BACKEND_PORT}:5001"
    networks:
      - shuffle
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps
      - ${SHUFFLE_FILE_LOCATION}:/shuffle-files
      #- ${SHUFFLE_OPENSEARCH_CERTIFICATE_FILE}:/shuffle-files/es_certificate
    env_file: .env
    environment:
      - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps
      - SHUFFLE_FILE_LOCATION=/shuffle-files
    restart: unless-stopped
    depends_on:
      - opensearch #Not necessary because dependancy is handled within the backend itself instead
      #- database
  opensearch:
    image: opensearchproject/opensearch:2
    hostname: shuffle-opensearch
    container_name: shuffle-opensearch
    environment:
      - bootstrap.memory_lock=true
      - "OPENSEARCH_JAVA_OPTS=-Xms4096m -Xmx4096m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
      - cluster.routing.allocation.disk.threshold_enabled=false
      - cluster.name=shuffle-cluster
      - node.name=shuffle-opensearch
      - discovery.seed_hosts=shuffle-opensearch
      - cluster.initial_master_nodes=shuffle-opensearch
      - node.store.allow_mmap=false
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 65536 
        hard: 65536
    volumes:
      - ${DB_LOCATION}:/usr/share/opensearch/data:rw
    ports:
      - 9200:9200
    networks:
      - shuffle
    restart: unless-stopped
networks:
  shuffle:
    driver: bridge
    #driver: overlay 

Shuffle/.env
# Default execution environment for workers
ORG_ID=Shuffle
ENVIRONMENT_NAME=Shuffle


# Remote github config for first load
SHUFFLE_DOWNLOAD_WORKFLOW_LOCATION=
SHUFFLE_DOWNLOAD_WORKFLOW_USERNAME=
SHUFFLE_DOWNLOAD_WORKFLOW_PASSWORD=
SHUFFLE_DOWNLOAD_WORKFLOW_BRANCH=

SHUFFLE_APP_DOWNLOAD_LOCATION=https://github.com/nusantara-self/Shuffle-apps
SHUFFLE_DOWNLOAD_AUTH_USERNAME=
SHUFFLE_DOWNLOAD_AUTH_PASSWORD=
SHUFFLE_DOWNLOAD_AUTH_BRANCH=
SHUFFLE_APP_FORCE_UPDATE=false
SSO_REDIRECT_URL=https://your-shuffle-domain-name.com


# User config for first load. Username & PW: min length 3
SHUFFLE_DEFAULT_USERNAME=
SHUFFLE_DEFAULT_PASSWORD=
SHUFFLE_DEFAULT_APIKEY=

# Local location of your app directory. Can't use ~/
# Files will get better at some point. Right now: local saving.
SHUFFLE_APP_HOTLOAD_FOLDER=./shuffle-apps
SHUFFLE_APP_HOTLOAD_LOCATION=./shuffle-apps
SHUFFLE_FILE_LOCATION=./shuffle-files

# Encryption modifier. This HAS to be set to encrypt any authentication being used in Shuffle. This is put together with other relevant values to ensure multiple parts are needed to decrypt. 
# If this key is lost or changed, you will have to reauthenticate all apps.
SHUFFLE_ENCRYPTION_MODIFIER=

# Other configs
BASE_URL=http://shuffle-backend:5001 
BACKEND_HOSTNAME=shuffle-backend
BACKEND_PORT=5001
FRONTEND_PORT=3001
FRONTEND_PORT_HTTPS=3443

# CHANGE THIS IF YOU WANT GOOD LOCAL EXECUTIONS:
OUTER_HOSTNAME=shuffle-backend
DB_LOCATION=./shuffle-database
DOCKER_API_VERSION=1.40

# Proxy configurations. SHUFFLE_PASS_WORKER_PROXY must be FALSE to not pass the proxy information to sub-apps.
# PS: It will skip proxy for 
HTTP_PROXY=
HTTPS_PROXY=
SHUFFLE_PASS_WORKER_PROXY=TRUE
SHUFFLE_PASS_APP_PROXY=FALSE
TZ=Europe/Amsterdam 					# Timezone-handler in Orborus, Worker and Apps
ORBORUS_CONTAINER_NAME=				# Used to FIND the containername. cgroup v2: issue 501

SHUFFLE_BASE_IMAGE_NAME=shuffle
SHUFFLE_BASE_IMAGE_REGISTRY=ghcr.io
SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-1.1.0"

# Used for auto-cleanup of containers. REALLY important at scale.
SHUFFLE_CONTAINER_AUTO_CLEANUP=true
SHUFFLE_ELASTIC=true
SHUFFLE_LOGS_DISABLED=false
SHUFFLE_CHAT_DISABLED=false
SHUFFLE_RERUN_SCHEDULE=300

# DATABASE CONFIGURATIONS
DATASTORE_EMULATOR_HOST=shuffle-database:8000
#SHUFFLE_OPENSEARCH_URL=http://shuffle-opensearch:9200
SHUFFLE_OPENSEARCH_URL=https://shuffle-opensearch:9200
SHUFFLE_OPENSEARCH_USERNAME=admin
SHUFFLE_OPENSEARCH_PASSWORD=admin
SHUFFLE_OPENSEARCH_CERTIFICATE_FILE=
SHUFFLE_OPENSEARCH_APIKEY=
SHUFFLE_OPENSEARCH_CLOUDID=
SHUFFLE_OPENSEARCH_PROXY=
SHUFFLE_OPENSEARCH_INDEX_PREFIX=
SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY=true

Worker / Orborus

On a separate server

Shuffle/docker-compose.yml
version: '3'
services:
  orborus:
    #build: ./functions/onprem/orborus
    image: ghcr.io/shuffle/shuffle-orborus:latest
    container_name: shuffle-orborus
    hostname: shuffle-orborus
    networks:
      - shuffle
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
    environment:
      - SHUFFLE_WORKER_VERSION=latest
      - SHUFFLE_APP_SDK_VERSION=latest
      - ORG_ID=${ENVIRONMENT_NAME}
      - ENVIRONMENT_NAME=${ENVIRONMENT_NAME}
      - BASE_URL=http://MANAGER-IP:5001
      - DOCKER_API_VERSION=1.40
      - SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME}
      - SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY}
      - SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX}
      - HTTP_PROXY=${HTTP_PROXY}
      - HTTPS_PROXY=${HTTPS_PROXY}
      - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY}
      - SHUFFLE_PASS_APP_PROXY=${SHUFFLE_PASS_APP_PROXY}
      - SHUFFLE_ORBORUS_EXECUTION_TIMEOUT=550
      - SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY=25
      - CLEANUP=true
    restart: unless-stopped
networks:
  shuffle:
    driver: bridge

Example of Shuffle/.env
# Default execution environment for workers
ORG_ID=Shuffle
ENVIRONMENT_NAME=OrborusPROD


# Remote github config for first load
SHUFFLE_DOWNLOAD_WORKFLOW_LOCATION=
SHUFFLE_DOWNLOAD_WORKFLOW_USERNAME=
SHUFFLE_DOWNLOAD_WORKFLOW_PASSWORD=
SHUFFLE_DOWNLOAD_WORKFLOW_BRANCH=

SHUFFLE_APP_DOWNLOAD_LOCATION=https://github.com/shuffle/python-apps
SHUFFLE_DOWNLOAD_AUTH_USERNAME=
SHUFFLE_DOWNLOAD_AUTH_PASSWORD=
SHUFFLE_DOWNLOAD_AUTH_BRANCH=
SHUFFLE_APP_FORCE_UPDATE=false

# User config for first load. Username & PW: min length 3
SHUFFLE_DEFAULT_USERNAME=
SHUFFLE_DEFAULT_PASSWORD=
SHUFFLE_DEFAULT_APIKEY=

# Local location of your app directory. Can't use ~/
# Files will get better at some point. Right now: local saving.
SHUFFLE_APP_HOTLOAD_FOLDER=./shuffle-apps
SHUFFLE_APP_HOTLOAD_LOCATION=./shuffle-apps
SHUFFLE_FILE_LOCATION=./shuffle-files

# Encryption modifier. This HAS to be set to encrypt any authentication being used in Shuffle. This is put together with other relevant values to ensure multiple parts are needed to decrypt. 
# If this key is lost or changed, you will have to reauthenticate all apps.
SHUFFLE_ENCRYPTION_MODIFIER=

# Other configs
BASE_URL=http://shuffle-backend:5001 
BACKEND_HOSTNAME=shuffle-backend
BACKEND_PORT=5001
FRONTEND_PORT=3001
FRONTEND_PORT_HTTPS=3443

# CHANGE THIS IF YOU WANT GOOD LOCAL EXECUTIONS:
OUTER_HOSTNAME=shuffle-backend
DB_LOCATION=./shuffle-database
DOCKER_API_VERSION=1.40

# Proxy configurations. SHUFFLE_PASS_WORKER_PROXY must be FALSE to not pass the proxy information to sub-apps.
# PS: It will skip proxy for 
HTTP_PROXY=
HTTPS_PROXY=
SHUFFLE_PASS_WORKER_PROXY=TRUE
SHUFFLE_PASS_APP_PROXY=FALSE
TZ=Europe/Amsterdam 					# Timezone-handler in Orborus, Worker and Apps
ORBORUS_CONTAINER_NAME=				# Used to FIND the containername. cgroup v2: issue 501

SHUFFLE_BASE_IMAGE_NAME=shuffle
SHUFFLE_BASE_IMAGE_REGISTRY=ghcr.io
SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-1.1.0"

# Used for auto-cleanup of containers. REALLY important at scale.
SHUFFLE_CONTAINER_AUTO_CLEANUP=true
SHUFFLE_ELASTIC=true
SHUFFLE_LOGS_DISABLED=false

# Cleans up any containers related to Shuffle that have been up for more than 600 seconds.
SHUFFLE_ORBORUS_EXECUTION_TIMEOUT=500

# Decides the max amount of workflows to concurrenly run. Defaults to 10.
# Example math: 10 workflows * WITH 10 apps / second = 110 containers per second.
# We recommend starting with 10 and going higher as need be.
SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY=30

# DATABASE CONFIGURATIONS
DATASTORE_EMULATOR_HOST=shuffle-database:8000
#SHUFFLE_OPENSEARCH_URL=http://shuffle-opensearch:9200
SHUFFLE_OPENSEARCH_URL=https://shuffle-opensearch:9200
SHUFFLE_OPENSEARCH_USERNAME=admin
SHUFFLE_OPENSEARCH_PASSWORD=admin
SHUFFLE_OPENSEARCH_CERTIFICATE_FILE=
SHUFFLE_OPENSEARCH_APIKEY=
SHUFFLE_OPENSEARCH_CLOUDID=
SHUFFLE_OPENSEARCH_PROXY=
SHUFFLE_OPENSEARCH_INDEX_PREFIX=
SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY=true

Issues

App malfunctioning

When an application is not working as expected, a few actions may need to be taken. (incomplete listing)

Delete the docker image

docker image ls
docker rmi frikky/shuffle:exampleapp_1.0.1

Shuffle

Resources​

Scripts​

Updating Shuffle​

Configuration​

Frontend / Backend / DB / "Manager"​

Worker / Orborus​

Issues​

App malfunctioning​

Delete the docker image​

Resources

Scripts

Updating Shuffle

Configuration

Frontend / Backend / DB / "Manager"

Worker / Orborus

Issues

App malfunctioning

Delete the docker image