
# create datset for demo
!parallel echo ::: {1..4} > n1.txt # 1 to 4, 1 elements per row       
!parallel echo ::: {a..z} > a2.txt # a to z, 1 elements per row

!parallel echo ::: {1..20} > n2.txt # 1 to 20, 1 elements per row   
!parallel echo ::: {a..d} > a1.txt # a to d, 1 elements per row

!parallel -N 2 echo ::: {a..z}  > a3.txt # a to z, 2 elements per row
!parallel -N 10 echo ::: {1..10000}  > n3.txt # 1 to 10000, 10 elements per row
!parallel -N 10 echo ::: {1..100}  > n4.txt # 1 to 10000, 10 elements per row


!echo "\nFrom CommandLine"
!parallel echo ::: a b c d
!echo "\nFrom Stdin"
!cat a1.txt | parallel echo 
## !cat a1.txt | parallel echo :::: -

From CommandLine
a
b
c
d

From Stdin
a
b
c
d


!echo "\n-N 5 for passing 5 elements at once"
!parallel -N 5 echo :::: a2.txt
!echo "\n-N 10 for passing 10 elements at once"
!parallel -N 10 echo :::: a2.txt

-N 5 for passing 5 elements at once
a b c d e
f g h i j
k l m n o
p q r s t
u v w x y
z

-N 10 for passing 10 elements at once
a b c d e f g h i j
k l m n o p q r s t
u v w x y z


!echo "\n-L 2 for passing 2 row at once"
!parallel -L 2 echo :::: a3.txt
!echo "\n-L 4 for passing 4 elements at once"
!parallel -L 4 echo :::: a3.txt

-L 2 for passing 2 row at once
a b c d
e f g h
i j k l
m n o p
q r s t
u v w x
y z

-L 4 for passing 4 elements at once
a b c d e f g h
i j k l m n o p
q r s t u v w x
y z


!echo "combination of multiple source"
!parallel echo ::: a b c d ::: a b c d
# !parallel echo :::: a.txt ::: a b c d
# !parallel -a a.txt -a a.txt echo
# !parallel echo :::: a.txt a.txt 
!echo "Get one element form each source at once (broadcasting)"
!parallel --xapply echo ::: a b c d ::: a b c d
# !parallel --xapply echo :::: a.txt ::: a b c d
# !parallel --xapply -a a.txt -a a.txt echo
# !parallel --xapply echo :::: a.txt a.txt

combination of multiple source
a a
a b
a c
a d
b a
b b
b c
b d
c a
c b
c c
c d
d a
d b
d c
d d
Get one element form each source at once (broadcasting)
a a
b b
c c
d d


%%time
!echo "\n--pipe for LoadBalance execute in recorder's order, with --block 10k for each block with 10kB"
!cat n3.txt | parallel --pipe -j4 --block 10k wc

--pipe for LoadBalance execute in recorder's order, with --block 10k for each block with 10kB
     200    2000   10000
     200    2000   10000
     222    2220    9993
     200    2000   10000
     178    1780    8901
CPU times: user 8.16 ms, sys: 11.5 ms, total: 19.6 ms
Wall time: 419 ms


%%time
!echo "--pipe --roundrobin for LoadBalance execute without recorder's order (faster), --block 10k for each block with 10kB"
!cat n3.txt | parallel --pipe -j4 --block 10k --roundrobin wc

--pipe --roundrobin for LoadBalance execute without recorder's order (faster), --block 10k for each block with 10kB
     200    2000   10000
     200    2000   10000
     200    2000   10000
     400    4000   18894
CPU times: user 7.19 ms, sys: 12.4 ms, total: 19.5 ms
Wall time: 395 ms


!parallel --jobs 4 echo pre-{}-post ::: A B C D E F G
!echo "\nNot repeat the background text  by each core"
!parallel --jobs 4 -m echo pre-{}-post ::: A B C D E F G
!echo "\nRepeat the background text by each core"
!parallel --jobs 4 -X echo pre-{}-post ::: A B C D E F G

pre-A-post
pre-B-post
pre-C-post
pre-D-post
pre-E-post
pre-F-post
pre-G-post

Not repeat the background text  by each core
pre-A B-post
pre-C D-post
pre-E F-post
pre-G-post

Repeat the background text by each core
pre-A-post pre-B-post
pre-C-post pre-D-post
pre-E-post pre-F-post
pre-G-post


!mkdir ./outdir
!parallel --files --tmpdir ./outdir ::: A B C
!rm -r outdir

outdir/parWVb7y.par
outdir/parNHXgt.par
outdir/parR6lxL.par


!echo "\n1d"
!parallel --results outdir echo ::: a1 a2
!tree outdir
!rm -r outdir 
!echo "\n1d with given header"
!parallel --header : --results outdir echo ::: A a1 a2
!tree outdir
!rm -r outdir

1d
outdir
└── 1
    ├── a1
    │   ├── seq
    │   ├── stderr
    │   └── stdout
    └── a2
        ├── seq
        ├── stderr
        └── stdout

3 directories, 6 files

1d with given header
outdir
└── A
    ├── a1
    │   ├── seq
    │   ├── stderr
    │   └── stdout
    └── a2
        ├── seq
        ├── stderr
        └── stdout

3 directories, 6 files


!echo "\n1d cross 1d"
!parallel --results outdir echo ::: a1 a2 ::: b1 b2
!tree outdir
!rm -r outdir 
!echo "\n1d cross 1d with given header"
!parallel --header : --results outdir echo ::: A a1 a2 ::: B b1 b2
!tree outdir
!rm -r outdir

1d cross 1d
outdir
└── 1
    ├── a1
    │   └── 2
    │       ├── b1
    │       │   ├── seq
    │       │   ├── stderr
    │       │   └── stdout
    │       └── b2
    │           ├── seq
    │           ├── stderr
    │           └── stdout
    └── a2
        └── 2
            ├── b1
            │   ├── seq
            │   ├── stderr
            │   └── stdout
            └── b2
                ├── seq
                ├── stderr
                └── stdout

9 directories, 12 files

1d cross 1d with given header
outdir
└── A
    ├── a1
    │   └── B
    │       ├── b1
    │       │   ├── seq
    │       │   ├── stderr
    │       │   └── stdout
    │       └── b2
    │           ├── seq
    │           ├── stderr
    │           └── stdout
    └── a2
        └── B
            ├── b1
            │   ├── seq
            │   ├── stderr
            │   └── stdout
            └── b2
                ├── seq
                ├── stderr
                └── stdout

9 directories, 12 files


!echo "\n1d broadcast 1d"
!parallel --results outdir --xapply echo ::: a1 a2 ::: b1 b2
!tree outdir
!rm -r outdir 
!echo "\n1d broadcast 1d with given header"
!parallel --header : --results outdir --xapply echo ::: A a1 a2 ::: B b1 b2
!tree outdir
!rm -r outdir

1d broadcast 1d
outdir
└── 1
    ├── a1
    │   └── 2
    │       └── b1
    │           ├── seq
    │           ├── stderr
    │           └── stdout
    └── a2
        └── 2
            └── b2
                ├── seq
                ├── stderr
                └── stdout

7 directories, 6 files

1d broadcast 1d with given header
outdir
└── A
    ├── a1
    │   └── B
    │       └── b1
    │           ├── seq
    │           ├── stderr
    │           └── stdout
    └── a2
        └── B
            └── b2
                ├── seq
                ├── stderr
                └── stdout

7 directories, 6 files


! echo '\n--verbose'
!cat n1.txt | parallel --verbose 'printf "%s-start\n%s" {} {};sleep {};printf "%s\n" -middle;echo {}-end' {}
! echo '\n--ungroup'
!parallel -j2 --ungroup 'printf "%s-start\n%s" {} {};sleep {};printf "%s\n" -middle;echo {}-end' :::: n1.txt
! echo '\n--linebuffer'
!parallel -j2 --linebuffer 'printf "%s-start\n%s" {} {};sleep {};printf "%s\n" -middle;echo {}-end' :::: n1.txt
! echo '\n--keep-order/-k '
!parallel -j2 -k 'printf "%s-start\n%s" {} {};sleep {};printf "%s\n" -middle;echo {}-end' :::: n1.txt

--verbose
printf "%s-start\n%s" 1 1;sleep 1;printf "%s\n" -middle;echo 1-end 1
printf "%s-start\n%s" 2 2;sleep 2;printf "%s\n" -middle;echo 2-end 2
printf "%s-start\n%s" 3 3;sleep 3;printf "%s\n" -middle;echo 3-end 3
printf "%s-start\n%s" 4 4;sleep 4;printf "%s\n" -middle;echo 4-end 4
1-start
1-middle
1-end 1
2-start
2-middle
2-end 2
3-start
3-middle
3-end 3
4-start
4-middle
4-end 4

--ungroup
1-start
12-start
2-middle
1-end
3-start
3-middle
2-end
4-start
4-middle
3-end
-middle
4-end

--linebuffer
1-start
2-start
1-middle
1-end
3-start
2-middle
2-end
4-start
3-middle
3-end
4-middle
4-end

--keep-order/-k 
1-start
1-middle
1-end
2-start
2-middle
2-end
3-start
3-middle
3-end
4-start
4-middle
4-end


!sysctl -a | grep machdep.cpu | grep count

machdep.cpu.core_count: 8
machdep.cpu.thread_count: 16


!echo '\nAssign One job for each thread by default'
!parallel -X echo run-{} ::: {1..1024} | wc -l
!echo '\nNot Parallel'
!parallel --jobs 1 -X echo run-{} ::: {1..1024} | wc -l
!echo '\nInitiate 2 job workder'
!parallel --jobs 2 -X echo run-{} ::: {1..1024} | wc -l
!echo '\nEach core take 200% utilization, so 2 for each thread'
!parallel --jobs 200% -X echo run-{} ::: {1..1024} | wc -l

Assign One job for each thread by default
      16

Not Parallel
       1

Initiate 2 job workder
       2

Each core take 200% utilization, so 2 for each thread
      32


!echo "Define sleep time between job to avoid thundering herd"
!parallel --delay 2.5 echo Starting {}\;date ::: 1 2 3

Define sleep time between job to avoid thundering herd
Starting 1
Thu Jun 30 20:59:40 EDT 2022
Starting 2
Thu Jun 30 20:59:43 EDT 2022
Starting 3
Thu Jun 30 20:59:45 EDT 2022


!echo "killed the job to end the job if pass defined time or percentage of median time"
!parallel --timeout 4.1 sleep {}\; echo {} ::: 2 4 6 8
!parallel --timeout 200% sleep {}\; echo {} ::: 2.1 2.2 3 7 2.3

killed the job to end the job if pass defined time or percentage of median time
2
4
parallel: Warning: This job was killed because it timed out:
parallel: Warning: sleep 6; echo 6
parallel: Warning: This job was killed because it timed out:
parallel: Warning: sleep 8; echo 8
2.1
2.2
2.3
3
parallel: Warning: This job was killed because it timed out:
parallel: Warning: sleep 7; echo 7


!echo "Show the progression information"
!parallel --progress sleep ::: 1 3 2 2 1 3 3 2 1
!parallel --eta sleep ::: 1 3 2 2 1 3 3 2 1

Show the progression information

Computers / CPU cores / Max jobs to run
1:local / 16 / 16

Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete
local:0/9/100%/0.3s 

Computers / CPU cores / Max jobs to run
1:local / 16 / 16

Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete
ETA: 0s Left: 0 AVG: 0.22s  local:0/9/100%/0.3s


!echo "Generate job log"
!parallel --joblog /tmp/log exit  ::: 1 2 3 0
!cat /tmp/log

Generate job log
Seq Host    Starttime   JobRuntime  Send    Receive Exitval Signal  Command
1   :   1656637203.405       0.005  0   0   1   0   exit 1
2   :   1656637203.409       0.005  0   0   2   0   exit 2
3   :   1656637203.413       0.004  0   0   3   0   exit 3
4   :   1656637203.417       0.004  0   0   0   0   exit 0


!echo "--retry-failed read failed job from the log "
!parallel --retry-failed --joblog /tmp/log exit  ::: 1 2 3 0 0 0
!echo "--resume-failed  read failed job from the cammandline "
!parallel --resume-failed --joblog /tmp/log exit  ::: 1 2 3 0 0 0

--retry-failed read failed job from the log 
--resume-failed  read failed job from the cammandline


!echo "--halt now stop generating new jobs and kill all running jobs"
!echo "--halt soon stop generating new jobs and wait for all running jobs to be done"
!echo "Halt when one of the jobs fails and has an exit code different from 0."
!parallel -j2 --halt soon,fail=1 echo {}\; exit {} ::: 0 0 1 2 3
!parallel -j2 --halt now,fail=1 echo {}\; exit {} ::: 0 0 1 2 3
!echo "\nHalt when one of the jobs success and has an exit code equal to 0."
!parallel -j2 --halt soon,success=1 echo {}\; exit {} ::: 1 2 3 0 4 5 6
!parallel -j2 --halt now,success=1 echo {}\; exit {} ::: 1 2 3 0 4 5 6
!echo "\nA percentage this percentage of the jobs must fail before GNU parallel stops spawning more jobs"
!parallel -j2 --halt soon,fail=20% echo {}\; exit {} ::: 0 1 2 3 4 5 6 7 8 9

--halt now stop generating new jobs and kill all running jobs
--halt soon stop generating new jobs and wait for all running jobs to be done
Halt when one of the jobs fails and has an exit code different from 0.
0
0
1
parallel: This job failed:
echo 1; exit 1
parallel: Starting no more jobs. Waiting for 1 jobs to finish.
2
parallel: This job failed:
echo 2; exit 2
0
0
1
parallel: This job failed:
echo 1; exit 1

Halt when one of the jobs success and has an exit code equal to 0.
1
2
3
0
parallel: This job succeeded:
echo 0; exit 0
parallel: Starting no more jobs. Waiting for 1 jobs to finish.
4
1
2
3
0
parallel: This job succeeded:
echo 0; exit 0

A percentage this percentage of the jobs must fail before GNU parallel stops spawning more jobs
0
1
parallel: This job failed:
echo 1; exit 1
2
parallel: This job failed:
echo 2; exit 2
parallel: Starting no more jobs. Waiting for 1 jobs to finish.
3
parallel: This job failed:
echo 3; exit 3


!parallel -k --retries 3 'echo tried {} >>/tmp/runs; echo completed {}; exit {}' ::: 1 2 0
!cat /tmp/runs

completed 1
completed 2
completed 0
tried 1
tried 2
tried 0
tried 1
tried 2
tried 1
tried 2
tried 1
tried 2
tried 1
tried 2
tried 1
tried 2
tried 0
tried 1
tried 2
tried 1
tried 2
tried 1
tried 2
tried 0


!echo "--memfree keep the certain amount of memory free"
!parallel --memfree 1G echo will run if more than 1 GB is ::: free

--memfree keep the certain amount of memory free
will run if more than 1 GB is free


!python3 -m jupyter nbconvert jupyter_GNU_Parallel.ipynb --to html

[NbConvertApp] Converting notebook jupyter_GNU_Parallel.ipynb to html
[NbConvertApp] Writing 633612 bytes to jupyter_GNU_Parallel.html

Standard file	File descriptor	Discription
stdin	0	usually the input from keyboard
stdout	1	standard output
stderr	2	standard error output

Command	Description
pgm > file	Output of pgm is redirected to file
pgm < file	Program pgm reads its input from file
pgm >> file	Output of pgm is appended to file
n > file	Output from stream with descriptor n redirected to file
n >> file	Output from stream with descriptor n appended to file
n >& m	Merges output from stream n with stream m
n <& m	Merges input from stream n with stream m
<< tag	Standard input comes from here through next tag at the start of line
\|	Takes output from one program, or process, and sends it to another

GNU Parallel

Weiquan Luo

2022-06-30

Table of Contents

Linux Background¶

Standard file¶

Input/Output Redirections¶

To Parallel: GNU Parallel¶

Input¶

format¶

-N¶

-L¶

--xapply¶

--pipe¶

Output¶

-m/-X¶

--files¶

--results¶

1d¶

1d cross 1d¶

1d broadcast 1d¶

Executing¶

--messaging format¶

--jobs/-j¶

--interactive¶

--delay¶

--timeout¶

--eta¶

-joblog¶

--resume-failed/--retry-failed¶

--halt¶

--retries¶

--memfree¶

Reference¶