feat(redis): use bitnami image

apps/dragonfly/.dockerignore

@@ -0,0 +1,2 @@
+.git
+fly.toml

apps/dragonfly/Dockerfile

@@ -0,0 +1,6 @@
+ARG REDIS_VERSION=7.2.5
+FROM bitnami/redis:${REDIS_VERSION}
+
+COPY start-redis-server.sh /usr/bin/start-redis-server.sh
+
+CMD ["/usr/bin/start-redis-server.sh"]

apps/dragonfly/Procfile

@@ -0,0 +1,2 @@
+redis: /usr/bin/start-redis-server.sh
+metrics: /usr/local/bin/redis_exporter -redis.addr localhost:6379 -web.listen-address ":9091"

apps/dragonfly/README.md

@@ -0,0 +1,48 @@
+The official repository for Running Redis on Fly.io. Find the accompanying Docker image at [flyio/redis](https://hub.docker.com/repository/docker/flyio/redis).
+
+## Usage
+
+This installation requires setting a password on Redis. To do that, run `fly secrets set REDIS_PASSWORD=mypassword` before deploying. Keep
+track of this password - it won't be visible again after deployment!
+
+If you need no customizations, you can deploy using the official Docker image. See `fly.toml` in this repository for an example to get started with.
+## Runtime requirements
+
+By default, this Redis installation will only accept connections on the private IPv6 network, on the standard port 6379.
+
+If you want to access it from the public internet, add a `[[services]]` section to your `fly.toml`. An example is included in this repo for accessing Redis on port 10000.
+
+
+We recommend adding persistent storage for Redis data. If you skip this step, data will be lost across deploys or restarts. For Fly apps, the volume needs to be in the same region as the app instances. For example:
+
+```cmd
+flyctl volumes create redis_server --region ord
+```
+```out
+      Name: redis_server
+    Region: ord
+   Size GB: 10
+Created at: 02 Nov 20 19:55 UTC
+```
+
+To connect this volume to the app, `fly.toml` includes a `[mounts]` entry.
+
+```
+[mounts]
+source      = "redis_server"
+destination = "/data"
+```
+
+When the app starts, that volume will be mounted on /data. 
+
+## Cutting a release
+
+If you have write access to this repo, you can ship a prerelease or full release with:
+
+```
+scripts/bump_version.sh
+```
+or
+```
+scripts/bump_version.sh prerel
+```

apps/dragonfly/fly.toml

@@ -1,13 +1,8 @@
 app = 'firecrawl-dragonfly'
 primary_region = 'iad'
 
-[experimental]
-  cmd = ['dragonfly','--logtostderr', '--cluster_mode=emulated', '--lock_on_hashtags', "--bind","::"]
-[build]
-  image = 'ghcr.io/dragonflydb/dragonfly'
-
 [[mounts]]
-  source = 'firecrawl_dragonfly'
+  source = 'firecrawl_redis'
   destination = '/data'
 
 [[services]]

apps/dragonfly/scripts/bump_version.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+ORIGIN=${ORIGIN:-origin}
+
+bump=${1:-patch}
+
+prerel=${2:-none}
+
+if [[ $bump == "prerel" ]]; then
+  bump="patch"
+  prerel="prerel"
+fi
+
+if [[ $(git status --porcelain) != "" ]]; then
+  echo "Error: repo is dirty. Run git status, clean repo and try again."
+  exit 1
+elif [[ $(git status --porcelain -b | grep -e "ahead" -e "behind") != "" ]]; then
+  echo "Error: repo has unpushed commits. Push commits to remote and try again."
+  exit 1
+fi
+
+BRANCH="$(git rev-parse --abbrev-ref HEAD)"
+if [[ "$prerel" == "prerel" && "$BRANCH" != "prerelease" ]]; then
+#  echo "❌ Sorry, you can only cut a pre-release from the 'prelease' branch"
+#  echo "Run 'git checkout prerelease && git pull origin prerelease' and try again."
+#  exit 1
+  echo "⚠️ Pre-releases should be cut from the 'prerelease' branch"
+  echo "Please make sure you're not overwriting someone else's prerelease!"
+  echo
+  read -p "Release anyway? " -n 1 -r
+  echo
+  if [[ $REPLY =~ ^[^Yy]$ ]]; then
+    echo Aborting.
+    exit 1
+  fi
+fi
+
+if [[ "$prerel" != "prerel" && "$BRANCH" != "main" ]]; then
+  echo "❌ Sorry, you can only cut a release from the 'main' branch"
+  echo "Run 'git checkout main && git pull origin main' and try again."
+  exit 1
+fi
+
+git fetch
+if [[ "$(git rev-parse HEAD 2>&1)" != "$(git rev-parse '@{u}' 2>&1)" ]]; then
+  echo "There are upstream commits that won't be included in this release."
+  echo "You probably want to exit, run 'git pull', then release."
+  echo
+  read -p "Release anyway? " -n 1 -r
+  echo
+  if [[ $REPLY =~ ^[^Yy]$ ]]; then
+    echo Aborting.
+    exit 1
+  fi
+fi
+
+dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+previous_version="$("$dir"/../scripts/version.sh -s)"
+
+if [[ $prerel == "prerel" ]]; then
+  prerelversion=$("$dir"/../scripts/semver get prerel "$previous_version")
+  if [[ $prerelversion == "" ]]; then
+    new_version=$("$dir"/../scripts/semver bump "$bump" "$previous_version")
+    new_version=$("$dir"/../scripts/semver bump prerel pre-1 "$new_version")
+  else
+    prerel=pre-$((${prerelversion#pre-} + 1))
+    new_version=$("$dir"/../scripts/semver bump prerel "$prerel" "$previous_version")
+  fi
+else
+  prerelversion=$("$dir"/../scripts/semver get prerel "$previous_version")
+  if [[ $prerelversion == "" ]]; then
+    new_version=$("$dir"/../scripts/semver bump "$bump" "$previous_version")
+  else
+    new_version=${previous_version//-$prerelversion/}
+  fi
+fi
+
+new_version="v$new_version"
+
+echo "Bumping version from v${previous_version} to ${new_version}"
+
+read -p "Are you sure? " -n 1 -r
+echo
+if [[ $REPLY =~ ^[Yy]$ ]]
+then
+  git tag -m "release ${new_version}" -a "$new_version" && git push "${ORIGIN}" tag "$new_version"
+  echo "done"
+fi

apps/dragonfly/scripts/semver

@@ -0,0 +1,200 @@
+#!/usr/bin/env bash
+
+set -o errexit -o nounset -o pipefail
+
+SEMVER_REGEX="^[vV]?(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)(\\-[0-9A-Za-z-]+(\\.[0-9A-Za-z-]+)*)?(\\+[0-9A-Za-z-]+(\\.[0-9A-Za-z-]+)*)?$"
+
+PROG=semver
+PROG_VERSION=2.1.0
+
+USAGE="\
+Usage:
+  $PROG bump (major|minor|patch|release|prerel <prerel>|build <build>) <version>
+  $PROG compare <version> <other_version>
+  $PROG get (major|minor|patch|release|prerel|build) <version>
+  $PROG --help
+  $PROG --version
+
+Arguments:
+  <version>  A version must match the following regex pattern:
+             \"${SEMVER_REGEX}\".
+             In english, the version must match X.Y.Z(-PRERELEASE)(+BUILD)
+             where X, Y and Z are positive integers, PRERELEASE is an optional
+             string composed of alphanumeric characters and hyphens and
+             BUILD is also an optional string composed of alphanumeric
+             characters and hyphens.
+
+  <other_version>  See <version> definition.
+
+  <prerel>  String that must be composed of alphanumeric characters and hyphens.
+
+  <build>   String that must be composed of alphanumeric characters and hyphens.
+
+Options:
+  -v, --version          Print the version of this tool.
+  -h, --help             Print this help message.
+
+Commands:
+  bump     Bump <version> by one of major, minor, patch, prerel, build
+           or a forced potentially conflicting version. The bumped version is
+           shown to stdout.
+
+  compare  Compare <version> with <other_version>, output to stdout the
+           following values: -1 if <other_version> is newer, 0 if equal, 1 if
+           older.
+
+  get      Extract given part of <version>, where part is one of major, minor,
+           patch, prerel, build."
+
+function error {
+  echo -e "$1" >&2
+  exit 1
+}
+
+function usage-help {
+  error "$USAGE"
+}
+
+function usage-version {
+  echo -e "${PROG}: $PROG_VERSION"
+  exit 0
+}
+
+function validate-version {
+  local version=$1
+  if [[ "$version" =~ $SEMVER_REGEX ]]; then
+    # if a second argument is passed, store the result in var named by $2
+    if [ "$#" -eq "2" ]; then
+      local major=${BASH_REMATCH[1]}
+      local minor=${BASH_REMATCH[2]}
+      local patch=${BASH_REMATCH[3]}
+      local prere=${BASH_REMATCH[4]}
+      local build=${BASH_REMATCH[6]}
+      eval "$2=(\"$major\" \"$minor\" \"$patch\" \"$prere\" \"$build\")"
+    else
+      echo "$version"
+    fi
+  else
+    error "version $version does not match the semver scheme 'X.Y.Z(-PRERELEASE)(+BUILD)'. See help for more information."
+  fi
+}
+
+function compare-version {
+  validate-version "$1" V
+  validate-version "$2" V_
+
+  # MAJOR, MINOR and PATCH should compare numerically
+  for i in 0 1 2; do
+    local diff=$((${V[$i]} - ${V_[$i]}))
+    if [[ $diff -lt 0 ]]; then
+      echo -1; return 0
+    elif [[ $diff -gt 0 ]]; then
+      echo 1; return 0
+    fi
+  done
+
+  # PREREL should compare with the ASCII order.
+  if [[ -z "${V[3]}" ]] && [[ -n "${V_[3]}" ]]; then
+    echo 1; return 0;
+  elif [[ -n "${V[3]}" ]] && [[ -z "${V_[3]}" ]]; then
+    echo -1; return 0;
+  elif [[ -n "${V[3]}" ]] && [[ -n "${V_[3]}" ]]; then
+    if [[ "${V[3]}" > "${V_[3]}" ]]; then
+      echo 1; return 0;
+    elif [[ "${V[3]}" < "${V_[3]}" ]]; then
+      echo -1; return 0;
+    fi
+  fi
+
+  echo 0
+}
+
+function command-bump {
+  local new; local version; local sub_version; local command;
+
+  case $# in
+    2) case $1 in
+        major|minor|patch|release) command=$1; version=$2;;
+        *) usage-help;;
+       esac ;;
+    3) case $1 in
+        prerel|build) command=$1; sub_version=$2 version=$3 ;;
+        *) usage-help;;
+       esac ;;
+    *) usage-help;;
+  esac
+
+  validate-version "$version" parts
+  # shellcheck disable=SC2154
+  local major="${parts[0]}"
+  local minor="${parts[1]}"
+  local patch="${parts[2]}"
+  local prere="${parts[3]}"
+  local build="${parts[4]}"
+
+  case "$command" in
+    major) new="$((major + 1)).0.0";;
+    minor) new="${major}.$((minor + 1)).0";;
+    patch) new="${major}.${minor}.$((patch + 1))";;
+    release) new="${major}.${minor}.${patch}";;
+    prerel) new=$(validate-version "${major}.${minor}.${patch}-${sub_version}");;
+    build) new=$(validate-version "${major}.${minor}.${patch}${prere}+${sub_version}");;
+    *) usage-help ;;
+  esac
+
+  echo "$new"
+  exit 0
+}
+
+function command-compare {
+  local v; local v_;
+
+  case $# in
+    2) v=$(validate-version "$1"); v_=$(validate-version "$2") ;;
+    *) usage-help ;;
+  esac
+
+  compare-version "$v" "$v_"
+  exit 0
+}
+
+
+# shellcheck disable=SC2034
+function command-get {
+    local part version
+
+    if [[ "$#" -ne "2" ]] || [[ -z "$1" ]] || [[ -z "$2" ]]; then
+        usage-help
+        exit 0
+    fi
+
+    part="$1"
+    version="$2"
+
+    validate-version "$version" parts
+    local major="${parts[0]}"
+    local minor="${parts[1]}"
+    local patch="${parts[2]}"
+    local prerel="${parts[3]:1}"
+    local build="${parts[4]:1}"
+
+    case "$part" in
+        major|minor|patch|release|prerel|build) echo "${!part}" ;;
+        *) usage-help ;;
+    esac
+
+    exit 0
+}
+
+case $# in
+  0) echo "Unknown command: $*"; usage-help;;
+esac
+
+case $1 in
+  --help|-h) echo -e "$USAGE"; exit 0;;
+  --version|-v) usage-version ;;
+  bump) shift; command-bump "$@";;
+  get) shift; command-get "$@";;
+  compare) shift; command-compare "$@";;
+  *) echo "Unknown arguments: $*"; usage-help;;
+esac

apps/dragonfly/scripts/version.sh

@@ -0,0 +1,5 @@
+ORIGIN=${ORIGIN:-origin}
+
+version=$(git fetch --tags "${ORIGIN}" &>/dev/null |  git -c "versionsort.prereleasesuffix=-pre" tag -l --sort=version:refname | grep -v dev | grep -vE '^v2$' | grep -vE '^v1$' | tail -n1 | cut -c 2-)
+
+echo "$version"

apps/dragonfly/start-redis-server.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -e
+
+sysctl vm.overcommit_memory=1 || true
+sysctl net.core.somaxconn=1024 || true
+
+PW_ARG=""
+if [[ ! -z "${REDIS_PASSWORD}" ]]; then
+  PW_ARG="--requirepass $REDIS_PASSWORD"
+fi
+
+# Set maxmemory-policy to 'allkeys-lru' for caching servers that should always evict old keys
+: ${MAXMEMORY_POLICY:="volatile-lru"}
+: ${APPENDONLY:="no"}
+: ${FLY_VM_MEMORY_MB:=512}
+if [ "${NOSAVE}" = "" ] ; then
+  : ${SAVE:="3600 1 300 100 60 10000"}
+fi
+# Set maxmemory to 10% of available memory
+MAXMEMORY=$(($FLY_VM_MEMORY_MB*80/100))
+
+mkdir /data/redis
+
+redis-server $PW_ARG \
+  --dir /data/redis \
+  --maxmemory "${MAXMEMORY}mb" \
+  --maxmemory-policy $MAXMEMORY_POLICY \
+  --appendonly $APPENDONLY \
+  --save "$SAVE"