1
req() {
2
  local url="$1" out="${2:-/dev/null}"
3
  local attempt=0 code=000 cache="UNKNOWN" wait=1
4
  local hdr="$TMPDIR/hdr.$$"
5
  while :; do
6
    : > "$hdr"
7
    code="$(curl -sS --compressed -m "$TIMEOUT" -A "$UA" \
8
              -D "$hdr" -o "$out" -w '%{http_code}' "$url" 2>/dev/null || echo 000)"
9
    cache="$(grep -i '^EO-Cache-Status:' "$hdr" 2>/dev/null \
10
              | head -n1 | sed -E 's/^[^:]+:[[:space:]]*//; s/\r$//')"
11
    [ -z "$cache" ] && cache="UNKNOWN"
12
    if [ "${code:0:1}" = "2" ] || [ "${code:0:1}" = "3" ]; then
13
      echo "$code|$cache"; return 0
14
    fi
15
    attempt=$((attempt+1))
16
    [ "$attempt" -gt "$RETRY" ] && { echo "$code|$cache"; return 1; }
17
    sleep "$wait"; wait=$((wait*2))
18
  done
19
}

1
extract_assets() {
2
  local html="$1"
3
  {
4
    grep -oE '(href|src)="[^"]+"' "$html" | sed -E 's/^(href|src)="//; s/"$//'
5
    grep -oE 'srcset="[^"]+"' "$html" | sed -E 's/^srcset="//; s/"$//' | tr ',' '\n' | awk '{print $1}'
6
  } | while read -r u; do
7
        [ -z "$u" ] && continue
8
        u="${u%%\#*}"; u="${u%%\?*}"
9
        case "$u" in
10
          //*)  u="https:$u" ;;
11
          /*)   u="$SITE$u" ;;
12
          http*) ;;
13
          *)    continue ;;
14
        esac
15
        case "$u" in "$SITE"*) ;; *) continue ;; esac
16
        case "$u" in
17
          *.css|*.js|*.png|*.jpg|*.jpeg|*.webp|*.gif|*.svg|*.ico|*.woff|*.woff2|*.ttf|*.otf|*.avif)
18
            echo "$u" ;;
19
        esac
20
      done
21
}

1
#!/usr/bin/env bash
2
# eo-warm.sh —— EdgeOne（免费版）缓存预热脚本（含缓存命中统计 + 二次验证）
3
#
4
# 原理：EdgeOne 免费版无预热 API，用「模拟真实访问」代替。
5
#       从 sitemap / robots.txt / 本地列表取 URL，走公网 GET 一遍，
6
#       命中边缘未命中则回源，边缘缓存被填充；页面引用的本站静态资源顺带预热。
7
#       通过响应头 EO-Cache-Status 判断 HIT / MISS / RefreshHit。
8
#
9
# 用法：
10
#   bash /opt/eo-warm.sh
11
#   EO_WARM_LIMIT=20 EO_WARM_CONCURRENCY=2 bash /opt/eo-warm.sh
12
#   EO_WARM_DRY_RUN=1 bash /opt/eo-warm.sh          # 只列 URL
13
#   EO_WARM_VERIFY=1 bash /opt/eo-warm.sh           # 预热后再跑一遍验证命中率
14
set -u
15

16
###################### 配置：直接改这里的值即可 ######################
17
SITE="https://x1anyu.cn"           # 站点地址，不带末尾斜杠；必须走公网域名
18
SITEMAP="/sitemap-index.xml"       # sitemap 入口
19
ROBOTS="/robots.txt"               # robots 兜底（SITEMAP 取不到时用）
20
LOCAL_LIST=""                      # 本地 URL 列表文件（可选，优先级最高）
21
DELAY=1                            # 每个请求间隔（秒）
22
CONCURRENCY=4                      # 并发数；1 = 串行。免费版建议 ≤4
23
RETRY=2                            # 失败重试次数
24
TIMEOUT=20                         # 单请求超时（秒）
25
LIMIT=0                            # 只预热前 N 条页面；0 = 全量
26
WARM_ASSETS=1                      # 1 = 顺带预热本站静态资源；0 = 只预热页面
27
LOG_LEVEL="info"                   # debug | info | warn | error
28
DRY_RUN=0                          # 1 = 只列 URL 不发请求
29
VERIFY=0                           # 1 = 预热后再跑一轮，统计 EO-Cache-Status
30
VERIFY_WAIT=5                      # 二次验证前等待秒数
31
UA="Mozilla/5.0 (compatible; SiteWarmer/1.0; +${SITE})"
32
############################################################
33

34
###################### 环境变量覆盖 ######################
35
SITE="${EO_WARM_SITE:-$SITE}"
36
SITEMAP="${EO_WARM_SITEMAP:-$SITEMAP}"
37
ROBOTS="${EO_WARM_ROBOTS:-$ROBOTS}"
38
LOCAL_LIST="${EO_WARM_LOCAL_LIST:-$LOCAL_LIST}"
39
DELAY="${EO_WARM_DELAY:-$DELAY}"
40
CONCURRENCY="${EO_WARM_CONCURRENCY:-$CONCURRENCY}"
41
RETRY="${EO_WARM_RETRY:-$RETRY}"
42
TIMEOUT="${EO_WARM_TIMEOUT:-$TIMEOUT}"
43
LIMIT="${EO_WARM_LIMIT:-$LIMIT}"
44
WARM_ASSETS="${EO_WARM_ASSETS:-$WARM_ASSETS}"
45
LOG_LEVEL="${EO_WARM_LOG_LEVEL:-$LOG_LEVEL}"
46
DRY_RUN="${EO_WARM_DRY_RUN:-$DRY_RUN}"
47
VERIFY="${EO_WARM_VERIFY:-$VERIFY}"
48
VERIFY_WAIT="${EO_WARM_VERIFY_WAIT:-$VERIFY_WAIT}"
49

50
SITE="${SITE%/}"                   # 去掉末尾斜杠
51
SITE_RE="$(printf '%s' "$SITE" | sed 's/\./\\./g')"   # 用于 grep 的正则
52

53
TMPDIR="$(mktemp -d)"
54
trap 'rm -rf "$TMPDIR"' EXIT
55

56
PAGES="$TMPDIR/pages.txt"
57
ASSETS="$TMPDIR/assets.txt"
58
RESULTS="$TMPDIR/results.txt"
59
VERIFY_RESULTS="$TMPDIR/verify.txt"
60
: > "$PAGES"; : > "$ASSETS"; : > "$RESULTS"; : > "$VERIFY_RESULTS"
61

62
###################### 日志 ######################
63
_level_num() {
64
  case "$1" in
65
    debug) echo 0;; info) echo 1;; warn) echo 2;; error) echo 3;; *) echo 1;;
66
  esac
67
}
68
LOG_NUM="$(_level_num "$LOG_LEVEL")"
69
log() {
70
  local lvl="$1"; shift
71
  [ "$(_level_num "$lvl")" -lt "$LOG_NUM" ] && return 0
72
  printf '[%s] [%s] %s\n' "$(date '+%F %T')" "$lvl" "$*" >&2
73
}
74

75
###################### 工具函数 ######################
76
# 单次请求：输出 "状态码|缓存状态"，重试走指数退避
77
req() {
78
  local url="$1" out="${2:-/dev/null}"
79
  local attempt=0 code=000 cache="UNKNOWN" wait=1
80
  local hdr="$TMPDIR/hdr.$$"
81

82
  while :; do
83
    : > "$hdr"
84
    code="$(curl -sS --compressed -m "$TIMEOUT" -A "$UA" \
85
              -D "$hdr" \
86
              -o "$out" -w '%{http_code}' "$url" 2>/dev/null || echo 000)"
87

88
    # 提取 EO-Cache-Status（不区分大小写，容忍空格）
89
    cache="$(grep -i '^EO-Cache-Status:' "$hdr" 2>/dev/null \
90
              | head -n1 \
91
              | sed -E 's/^[^:]+:[[:space:]]*//; s/\r$//')"
92
    [ -z "$cache" ] && cache="UNKNOWN"
93

94
    if [ "${code:0:1}" = "2" ] || [ "${code:0:1}" = "3" ]; then
95
      echo "$code|$cache"; return 0
96
    fi
97
    attempt=$((attempt+1))
98
    if [ "$attempt" -gt "$RETRY" ]; then
99
      echo "$code|$cache"; return 1
100
    fi
101
    sleep "$wait"
102
    wait=$((wait*2))
103
  done
104
}
105

106
# 仅取缓存状态（验证用，不关心 body）
107
req_cache_only() {
108
  local url="$1"
109
  local hdr="$TMPDIR/hdr.$$"
110
  : > "$hdr"
111
  curl -sS --compressed -m "$TIMEOUT" -A "$UA" \
112
    -D "$hdr" -o /dev/null "$url" 2>/dev/null || true
113
  local cache
114
  cache="$(grep -i '^EO-Cache-Status:' "$hdr" 2>/dev/null \
115
            | head -n1 \
116
            | sed -E 's/^[^:]+:[[:space:]]*//; s/\r$//')"
117
  [ -z "$cache" ] && cache="UNKNOWN"
118
  echo "$cache"
119
}
120

121
# 从 HTML 里提取本站静态资源
122
extract_assets() {
123
  local html="$1"
124
  {
125
    grep -oE '(href|src)="[^"]+"' "$html" | sed -E 's/^(href|src)="//; s/"$//'
126
    grep -oE 'srcset="[^"]+"'     "$html" | sed -E 's/^srcset="//; s/"$//' \
127
      | tr ',' '\n' | awk '{print $1}'
128
  } | while read -r u; do
129
        [ -z "$u" ] && continue
130
        u="${u%%\#*}"; u="${u%%\?*}"
131
        case "$u" in
132
          //*)  u="https:$u" ;;
133
          /*)   u="$SITE$u" ;;
134
          http*) ;;
135
          *)    continue ;;
136
        esac
137
        case "$u" in "$SITE"*) ;; *) continue ;; esac
138
        case "$u" in
139
          *.css|*.js|*.png|*.jpg|*.jpeg|*.webp|*.gif|*.svg|*.ico|*.woff|*.woff2|*.ttf|*.otf|*.avif)
140
            echo "$u" ;;
141
        esac
142
      done
143
}
144

145
###################### 1. 采集页面 URL ######################
146
log info "开始预热：$SITE"
147

148
# 1.1 本地列表优先
149
if [ -n "$LOCAL_LIST" ] && [ -f "$LOCAL_LIST" ]; then
150
  log info "从本地列表读取：$LOCAL_LIST"
151
  grep -E '^https?://' "$LOCAL_LIST" >> "$PAGES"
152
fi
153

154
# 1.2 sitemap（递归），失败时兜底 robots.txt
155
if [ ! -s "$PAGES" ]; then
156
  STACK=("${SITE}${SITEMAP}")
157
  DEPTH_MAP=("0")
158
  while [ "${#STACK[@]}" -gt 0 ]; do
159
    url="${STACK[0]}"; depth="${DEPTH_MAP[0]}"
160
    STACK=("${STACK[@]:1}"); DEPTH_MAP=("${DEPTH_MAP[@]:1}")
161
    [ "$depth" -gt 3 ] && { log debug "sitemap 递归过深，跳过：$url"; continue; }
162

163
    body="$(curl -sS --compressed -m "$TIMEOUT" -A "$UA" "$url" 2>/dev/null || true)"
164
    if [ -z "$body" ]; then
165
      log warn "无法拉取 sitemap：$url"
166
      continue
167
    fi
168
    if printf '%s' "$body" | grep -q "<sitemapindex"; then
169
      while IFS= read -r sub; do
170
        [ -z "$sub" ] && continue
171
        STACK+=("$sub"); DEPTH_MAP+=("$((depth+1))")
172
      done < <(printf '%s' "$body" | grep -oE '<loc>[^<]+</loc>' | sed -E 's#</?loc>##g')
173
    else
174
      printf '%s' "$body" | grep -oE '<loc>[^<]+</loc>' | sed -E 's#</?loc>##g' >> "$PAGES"
175
    fi
176
  done
177
fi
178

179
# 1.3 robots.txt 兜底
180
if [ ! -s "$PAGES" ]; then
181
  log warn "sitemap 无结果，尝试 robots.txt 兜底"
182
  body="$(curl -sS --compressed -m "$TIMEOUT" -A "$UA" "${SITE}${ROBOTS}" 2>/dev/null || true)"
183
  if [ -n "$body" ]; then
184
    while IFS= read -r sub; do
185
      [ -z "$sub" ] && continue
186
      sub_body="$(curl -sS --compressed -m "$TIMEOUT" -A "$UA" "$sub" 2>/dev/null || true)"
187
      printf '%s' "$sub_body" | grep -oE '<loc>[^<]+</loc>' | sed -E 's#</?loc>##g' >> "$PAGES"
188
    done < <(printf '%s' "$body" | awk -F': *' 'tolower($1)=="sitemap" {print $2}')
189
  fi
190
fi
191

192
# 去重 + 只保留本站
193
if [ -s "$PAGES" ]; then
194
  grep -E "^${SITE_RE}" "$PAGES" | sort -u > "$PAGES.clean" || true
195
  mv "$PAGES.clean" "$PAGES"
196
fi
197

198
TOTAL=$(wc -l < "$PAGES" 2>/dev/null || echo 0)
199
if [ "$TOTAL" -eq 0 ]; then
200
  log error "未采集到任何页面 URL，退出"
201
  exit 1
202
fi
203
if [ "$LIMIT" -gt 0 ] && [ "$TOTAL" -gt "$LIMIT" ]; then
204
  head -n "$LIMIT" "$PAGES" > "$PAGES.lim" && mv "$PAGES.lim" "$PAGES"
205
fi
206
COUNT=$(wc -l < "$PAGES")
207
log info "采集完成：sitemap/robots 共 $TOTAL 条，本次预热 $COUNT 条（并发 $CONCURRENCY，重试 $RETRY）"
208

209
if [ "$DRY_RUN" = "1" ]; then
210
  log info "DRY-RUN 模式，仅输出 URL："
211
  cat "$PAGES"
212
  exit 0
213
fi
214

215
###################### 2. 预热页面（并发） ######################
216
warm_page() {
217
  local page="$1"
218
  local body="$TMPDIR/page.$$.html"
219
  local result code cache
220
  result="$(req "$page" "$body")" || true
221
  code="${result%%|*}"
222
  cache="${result##*|}"
223
  printf 'PAGE\t%s\t%s\t%s\n' "$code" "$cache" "$page" >> "$RESULTS"
224
  log info "PAGE $code [$cache]  $page"
225

226
  if [ "$WARM_ASSETS" = "1" ] && [ "${code:0:1}" = "2" ]; then
227
    extract_assets "$body" >> "$ASSETS"
228
  fi
229
  rm -f "$body"
230
  [ "$DELAY" -gt 0 ] && sleep "$DELAY"
231
}
232
export -f warm_page req extract_assets log _level_num
233
export UA TIMEOUT RETRY DELAY RESULTS ASSETS TMPDIR WARM_ASSETS SITE LOG_NUM
234

235
log info "开始预热页面…"
236
if [ "$CONCURRENCY" -le 1 ]; then
237
  while IFS= read -r p; do [ -n "$p" ] && warm_page "$p"; done < "$PAGES"
238
else
239
  # shellcheck disable=SC2016
240
  xargs -a "$PAGES" -I{} -P "$CONCURRENCY" bash -c 'warm_page "$@"' _ {}
241
fi
242

243
###################### 3. 预热静态资源（去重 + 并发） ######################
244
if [ "$WARM_ASSETS" = "1" ] && [ -s "$ASSETS" ]; then
245
  sort -u "$ASSETS" > "$ASSETS.clean" && mv "$ASSETS.clean" "$ASSETS"
246
  ASSET_COUNT=$(wc -l < "$ASSETS")
247
  log info "开始预热 $ASSET_COUNT 个静态资源…"
248

249
  warm_asset() {
250
    local a="$1" result code cache
251
    result="$(req "$a")" || true
252
    code="${result%%|*}"
253
    cache="${result##*|}"
254
    printf 'ASSET\t%s\t%s\t%s\n' "$code" "$cache" "$a" >> "$RESULTS"
255
    log info "ASSET $code [$cache]  $a"
256
    [ "$DELAY" -gt 0 ] && sleep "$DELAY"
257
  }
258
  export -f warm_asset
259
  if [ "$CONCURRENCY" -le 1 ]; then
260
    while IFS= read -r a; do [ -n "$a" ] && warm_asset "$a"; done < "$ASSETS"
261
  else
262
    xargs -a "$ASSETS" -I{} -P "$CONCURRENCY" bash -c 'warm_asset "$@"' _ {}
263
  fi
264
fi
265

266
###################### 4. 预热结果统计 ######################
267
log info "预热完成，统计如下："
268
if [ -s "$RESULTS" ]; then
269
  awk -F'\t' '
270
    { total++; cls=substr($2,1,1); cache=$3
271
      if (cls=="2") ok++; else if (cls=="3") redir++; else if (cls=="4") c4++; else if (cls=="5") c5++; else fail++
272
      cstat[cache]++
273
      key=$1" "$2" ["$3"]"; cnt[key]++
274
    }
275
    END {
276
      printf "  总计: %d\n", total
277
      printf "  成功(2xx): %d  重定向(3xx): %d  客户端错(4xx): %d  服务端错(5xx): %d  失败: %d\n", ok, redir, c4, c5, fail
278
      print  "  --- 缓存状态分布（首次预热，MISS 属正常）---"
279
      for (c in cstat) printf "  %-12s %d\n", c, cstat[c]
280
      print  "  --- PAGE/ASSET 明细 ---"
281
      for (k in cnt) printf "  %-32s %d\n", k, cnt[k]
282
    }
283
  ' "$RESULTS" | sort
284
  echo "  --- 非 2xx 明细 ---" >&2
285
  awk -F'\t' '$2 !~ /^2/ {print "  "$2" ["$3"] "$4}' "$RESULTS" >&2 || true
286
else
287
  log warn "无请求记录"
288
fi
289

290
###################### 5. 二次验证：缓存命中率 ######################
291
if [ "$VERIFY" = "1" ]; then
292
  log info "等待 ${VERIFY_WAIT}s 让边缘缓存生效，然后二次验证…"
293
  sleep "$VERIFY_WAIT"
294

295
  # 汇总所有已请求的 URL（页面 + 资源）
296
  ALL_URLS="$TMPDIR/all_urls.txt"
297
  awk -F'\t' '{print $4}' "$RESULTS" | sort -u > "$ALL_URLS"
298
  V_TOTAL=$(wc -l < "$ALL_URLS")
299
  log info "开始验证 $V_TOTAL 个 URL 的 EO-Cache-Status…"
300

301
  verify_one() {
302
    local u="$1" c
303
    c="$(req_cache_only "$u")"
304
    printf '%s\t%s\n' "$c" "$u" >> "$VERIFY_RESULTS"
305
  }
306
  export -f verify_one req_cache_only
307
  export VERIFY_RESULTS
308

309
  if [ "$CONCURRENCY" -le 1 ]; then
310
    while IFS= read -r u; do [ -n "$u" ] && verify_one "$u"; done < "$ALL_URLS"
311
  else
312
    xargs -a "$ALL_URLS" -I{} -P "$CONCURRENCY" bash -c 'verify_one "$@"' _ {}
313
  fi
314

315
  log info "验证完成，缓存命中统计："
316
  if [ -s "$VERIFY_RESULTS" ]; then
317
    awk -F'\t' '
318
      { total++; cstat[$1]++ }
319
      END {
320
        printf "  验证总数: %d\n", total
321
        for (c in cstat) {
322
          pct = (total>0) ? (cstat[c]*100.0/total) : 0
323
          printf "  %-12s %d (%.1f%%)\n", c, cstat[c], pct
324
        }
325
      }
326
    ' "$VERIFY_RESULTS" | sort
327

328
    echo "  --- 非 HIT 明细 ---" >&2
329
    awk -F'\t' '$1 != "HIT" {print "  "$1"\t"$2}' "$VERIFY_RESULTS" >&2 || true
330
  else
331
    log warn "无验证记录"
332
  fi
333
fi
334

335
log info "全部完成 ✅"