Merge pull request #492 from mendableai/nsc/amazon

Fix AMZN | Removal of redis alerts

.github/workflows/check-redis.yml

@@ -1,20 +0,0 @@
-name: Check Redis
-on:
-  schedule:
-    - cron: '*/5 * * * *'
-
-env:
-  BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
-    
-jobs:
-  clean-jobs:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Send GET request to check queues
-        run: |
-          response=$(curl --write-out '%{http_code}' --silent --output /dev/null --max-time 180 https://api.firecrawl.dev/admin/${{ secrets.BULL_AUTH_KEY }}/redis-health)
-          if [ "$response" -ne 200 ]; then
-            echo "Failed to check queues. Response: $response"
-            exit 1
-          fi
-          echo "Successfully checked queues. Response: $response"

apps/api/src/scraper/WebScraper/utils/custom/website_params.ts

@@ -232,4 +232,14 @@ export const urlSpecificParams = {
       }
     },
   },
+  "amazon.com":{
+    defaultScraper: "fire-engine",
+    params:{
+      fireEngineOptions:{
+        mobileProxy: true,
+        method: "get",
+        engine: "chrome-cdp",
+      },
+    },
+  },
 };