TrendRadar/.github/workflows/crawler.yml

name: Hot News Crawler
on:
  schedule:
    - cron: "0 * * * *"
  workflow_dispatch:

concurrency:
  group: crawler-main-branch
  cancel-in-progress: true

permissions:
  contents: write

jobs:
  crawl:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
        with:
          ref: main
          fetch-depth: 0
          clean: true
          
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"
          
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt
          
      - name: Verify required files
        run: |
          echo "🔍 检查必需的配置文件..."
          if [ ! -f config/config.yaml ]; then
            echo "❌ 错误: config/config.yaml 文件不存在"
            exit 1
          fi
          if [ ! -f config/frequency_words.txt ]; then
            echo "❌ 错误: config/frequency_words.txt 文件不存在"
            exit 1
          fi
          echo "✅ 配置文件检查通过"
          
      - name: Run crawler
        env:
          FEISHU_WEBHOOK_URL: ${{ secrets.FEISHU_WEBHOOK_URL }}
          TELEGRAM_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }}
          TELEGRAM_CHAT_ID: ${{ secrets.TELEGRAM_CHAT_ID }}
          DINGTALK_WEBHOOK_URL: ${{ secrets.DINGTALK_WEBHOOK_URL }}
          WEWORK_WEBHOOK_URL: ${{ secrets.WEWORK_WEBHOOK_URL }}
          WEWORK_MSG_TYPE: ${{ secrets.WEWORK_MSG_TYPE }}
          EMAIL_FROM: ${{ secrets.EMAIL_FROM }}
          EMAIL_PASSWORD: ${{ secrets.EMAIL_PASSWORD }}
          EMAIL_TO: ${{ secrets.EMAIL_TO }}
          EMAIL_SMTP_SERVER: ${{ secrets.EMAIL_SMTP_SERVER }}
          EMAIL_SMTP_PORT: ${{ secrets.EMAIL_SMTP_PORT }}
          NTFY_TOPIC: ${{ secrets.NTFY_TOPIC }}
          NTFY_SERVER_URL: ${{ secrets.NTFY_SERVER_URL }}
          NTFY_TOKEN: ${{ secrets.NTFY_TOKEN }}
          BARK_URL: ${{ secrets.BARK_URL }}
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
          GITHUB_ACTIONS: true
        run: python main.py
        
      - name: Commit and push if changes
        run: |
          git config --global user.name 'GitHub Actions'
          git config --global user.email 'actions@github.com'
          
          echo "🔄 Syncing with remote..."
          git fetch origin main
          
          # 保存当前更改
          git stash --include-untracked || echo "Nothing to stash"
          
          # 同步到远程最新
          git reset --hard origin/main
          
          # 恢复本次更改
          git stash pop || echo "Nothing to pop"
          
          git add -A
          
          if git diff --quiet && git diff --staged --quiet; then
            echo "📭 No changes to commit"
            exit 0
          fi
          
          echo "📝 Committing changes..."
          TIMESTAMP=$(TZ=Asia/Shanghai date "+%Y-%m-%d %H:%M:%S")
          git commit -m "Auto update by GitHub Actions at $TIMESTAMP"
          
          echo "⬆️ Pushing changes with retry..."
          for i in {1..5}; do
            git pull --rebase origin main && git push origin main && {
              echo "✅ Successfully pushed on attempt $i"
              exit 0
            }
            echo "⚠️ Attempt $i/$i failed, waiting $((i*3)) seconds..."
            sleep $((i * 3))
          done
          
          echo "❌ Failed to push after 5 attempts"
          exit 1