GNU Parallel批量处理：从串行到并行

1
# 串行——一个接一个，CPU闲得发慌
2
for r1 in *_R1.fastq.gz; do
3
    r2="${r1/_R1/_R2}"
4
    sample=$(basename "$r1" _R1.fastq.gz)
5
    fastp -i "$r1" -I "$r2" \
6
        -o "clean/${sample}_R1.fastq.gz" \
7
        -O "clean/${sample}_R2.fastq.gz" \
8
        -w 4
9
done

1
ls *_R1.fastq.gz | parallel -j4 \
2
    'r1={}; r2=$(echo {} | sed "s/_R1/_R2/"); sample=$(basename {} _R1.fastq.gz);
3
     fastp -i "$r1" -I "$r2" -o "clean/${sample}_R1.fastq.gz" -O "clean/${sample}_R2.fastq.gz" -w 2'

1
run_bwa() {
2
    local r1="$1"
3
    local sample=$(basename "$r1" _R1.fastq.gz)
4
    local r2="${r1/_R1/_R2}"
5

6
    bwa mem -t 4 -R "@RG\tID:${sample}\tSM:${sample}" \
7
        /opt/refs/hg38.fa "$r1" "$r2" | \
8
        samtools sort -@ 2 -o "bam/${sample}.sorted.bam"
9

10
    samtools index "bam/${sample}.sorted.bam"
11
}
12

13
export -f run_bwa
14
ls raw/*_R1.fastq.gz | parallel -j2 run_bwa {}

1
# 从 accession list 并行下载
2
cat sra_ids.txt | parallel -j4 \
3
    'prefetch {} && fasterq-dump --split-3 {} -O fastq/ && rm -rf ~/ncbi/public/sra/{}'

1
# 对20个基因跑差异分析
2
parallel -j8 'Rscript run_deseq.R --gene {} --output results/{}.csv' ::: gene1 gene2 ... gene20
3

4
# 对多个癌症类型跑 Python 特征筛选
5
parallel -j4 'python select_features.py --cancer {} --output features/{}.tsv' ::: BRCA LUAD COAD

1
# 把任务分发到3台机器，每台跑3个
2
parallel -S node1,node2,node3 -j9 \
3
    'bwa mem -t 2 /opt/refs/hg38.fa {} > {/.}.sam' ::: *.fastq.gz

1
# 错误
2
run_fastp() { ... }
3
parallel run_fastp ::: *.fastq.gz
4
# /bin/bash: run_fastp: command not found
5

6
# 正确：需要 export -f
7
export -f run_fastp
8
parallel run_fastp ::: *.fastq.gz

1
# 8核机器，正确配置
2
parallel -j4 配合 fastp -w 2  → 4×2=8 ✓
3
parallel -j2 配合 bwa -t 4    → 2×4=8 ✓
4

5
# 错误配置
6
parallel -j8 配合 fastp -w 4  → 8×4=32 ✗

1
# 单引号里 $HOME 不会被展开
2
parallel 'echo $HOME' ::: 1 2 3
3
# 输出空
4

5
# 解决方法1：双引号+转义
6
parallel "echo \$HOME" ::: 1 2 3
7

8
# 解决方法2（最推荐）：用函数封装，完全避开引用问题
9
my_echo() { echo "$HOME - $1"; }
10
export -f my_echo
11
parallel my_echo ::: a b c

1
# 管道默认按空白符分割，文件名含空格会截断
2
ls *.fastq.gz | parallel gzip -t {}
3
# "sample A_R1.fastq.gz" → {} 只拿到 "sample"
4

5
# 解决方案：NULL 分隔符
6
find . -name "*.fastq.gz" -print0 | parallel -0 gzip -t {}

1
# 检查系统日志确认是否被 OOM
2
dmesg | grep -i "out of memory"
3
journalctl -k | grep oom
4

5
# 如果确认是 OOM，降低 -j 或加内存

1
# 默认所有任务的 stdout/stderr 混在一起，完全没法看
2
parallel fastp -i {} ::: *.fastq.gz
3

4
# 解决：每个任务重定向到独立日志
5
parallel 'fastp -i {} > logs/{/.}.log 2>&1' ::: *.fastq.gz
6

7
# 或者用 --results 自动分目录
8
parallel --results output_logs fastp -i {} ::: *.fastq.gz
9
# 生成 output_logs/1/sample1/stdout, output_logs/1/sample1/stderr ...

1
# 错误写法：retries 在 halt 之后
2
parallel --halt now,fail=1 --retries 3 ...
3
# 失败任务会重试3次才触发 halt——浪费时间
4

5
# 正确写法：retries 在 halt 之前
6
parallel --retries 3 --halt now,fail=1 ...

1
parallel --timeout 1800 --retries 1 --halt now,fail=1 ...