forked from famfigueiredo/my-bash-reference
-
Notifications
You must be signed in to change notification settings - Fork 0
/
useful_bash_commands.sh
237 lines (157 loc) · 8.9 KB
/
useful_bash_commands.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
https://explainshell.com/
### Useful bash commands ###
# Cut/Paste current command behind cursor
Ctrl + U, Ctrl + Y
### SSH keys
ls ~/.ssh/*.pub # list all your public keys
ssh-agent sh -c 'ssh-add; ssh-add -l' # RSA fingerprint
ssh-agent sh -c 'ssh-add; ssh-add -L' # public key
pbcopy < ~/.ssh/id_rsa.pub # copy RSA key to notepad
ssh [email protected] # login to Spygene
### File transfer from remote to local
rsync -havP $src $dst
# rsync arguments ... -h: human-readable format; -P: combines --progress and --partial; -n: dry run; -a: archive mode; -v: verbose; -z: compression; -e: specify the remote shell to use (i.e. ssh); --info: progress bar
rsync -havP . [email protected]:/data/ffi007/01_quantseq/03_data/january2020/02_unzipped-source-files/02_lane2-december2019/02_demuxed-files
### Kill all screen sessions
killall screen # DO NOT SUDO! #
### Check HD space
du -lh # directory size
df -lh # system-wide space availability
du -h -d1 # level-specific directory size
# Another way to get a report about the disk usage of the first-level subdirectories is to use the --max-depth option:
du -h --max-depth=1 /data
### Refresh path
hash -r
### Check versions of software in your PATH
which -a software
### In the case of needing to use an older version of Java #
sudo apt-get update
sudo apt-get openjdk-8-jre # installing older version of Java #
sudo update-alternatives --config java # adding older version of Java to root path #
### Creating soft links to data #
ln -s /PATH/TO/ORIGINAL/FOLDER/*.fq.gz . # This creates a soft link of all the .fq.gz files in original directory, in current directory #
### Changing directory ownership #
sudo chown -R ffi007:ffi007 . # Changing ownership of all current directories (-R) #
sudo chown -R ffi007:science folder/
### Permission numbers. Order goes user/group/users #
0 = ---
1 = --x
2 = -w-
3 = -wx
4 = r-
5 = r-x
6 = rw-
7 = rwx
### Remember to change permissions for bash script execution #
chmod 755 bashcript.sh
chmod +x bashcript.sh # to make it executable
### And for the filelist.txt #
chmod 755 filelist.txt
### Find a file #
find /data -name uniprot_modelfish_backtranseq.fasta # find the fasta file in the /data directory and all sub-directories #
### Print line X of GFF file #
sed -n Xp GFF_FILE.gff
### Selecting lines of interest in a GFF file #
grep transcript_id inputGFFfile > outputGFFfile # all lines containing 'transcript_id' #
grep -A20 -B20 target file.txt # 20 lines before and after the target
### Counting lines in a file #
wc -l someGFFfile.gff
### Show the first line (column headers) in an organized manner #
cat * | head -n 1 | tr “\t” “\n” | nl
### Selecting lines from a txt file #
awk '$2 > 0' FILE.txt > SOMEOUTPUTFILE.txt # Selecting lines where column 2 has a value > 0 #
### Print selected columns from a txt file #
awk '{print $x,$y}' inputfile > outputfile # For x and y columns #
### Reading a few lines from a .gz file #
zcat file.fq.gz | head
### Read count in fq.gz file #
zcat file.fq.gz | echo $((`wc -l`/4))
### Screen management #
screen -r -d sessioname # reopen attached sessions #
screen -S sessioname # create session with custom name #
screen -X -S sessioname kill # kill screen session #
### Rename files large number of files #
rename 's/abc/xyz/' *.bam # Rename .bam files where 'abc' shows up, change it to 'xyz' #
### Find a specific file/program #
find / -iname multiqc # might have to sudo #
### Indexing bam file using samtools #
samtools index bamfile.bam
### View bam file #
samtools view -h bamfile.bam
### To write the output of a command to a file, there are basically 10 commonly used ways.
# Overview:
# || visible in terminal || visible in file || existing
# Syntax || StdOut | StdErr || StdOut | StdErr || file
# ==========++==========+==========++==========+==========++===========
# > || no | yes || yes | no || overwrite
# >> || no | yes || yes | no || append
# || | || | ||
# 2> || yes | no || no | yes || overwrite
# 2>> || yes | no || no | yes || append
# || | || | ||
# &> || no | no || yes | yes || overwrite
# &>> || no | no || yes | yes || append
# || | || | ||
# | tee || yes | yes || yes | no || overwrite
# | tee -a || yes | yes || yes | no || append
# || | || | ||
# n.e. (*) || yes | yes || no | yes || overwrite
# n.e. (*) || yes | yes || no | yes || append
# || | || | ||
# |& tee || yes | yes || yes | yes || overwrite
# |& tee -a || yes | yes || yes | yes || append
# List:
command > output.txt
# The standard output stream will be redirected to the file only, it will not be visible in the terminal. If the file already exists, it gets overwritten.
command >> output.txt
# The standard output stream will be redirected to the file only, it will not be visible in the terminal. If the file already exists, the new data will get appended to the end of the file.
command 2> output.txt
# The standard error stream will be redirected to the file only, it will not be visible in the terminal. If the file already exists, it gets overwritten.
command 2>> output.txt
# The standard error stream will be redirected to the file only, it will not be visible in the terminal. If the file already exists, the new data will get appended to the end of the file.
command &> output.txt
# Both the standard output and standard error stream will be redirected to the file only, nothing will be visible in the terminal. If the file already exists, it gets overwritten.
command &>> output.txt
# Both the standard output and standard error stream will be redirected to the file only, nothing will be visible in the terminal. If the file already exists, the new data will get appended to the end of the file..
command | tee output.txt
# The standard output stream will be copied to the file, it will still be visible in the terminal. If the file already exists, it gets overwritten.
command | tee -a output.txt
# The standard output stream will be copied to the file, it will still be visible in the terminal. If the file already exists, the new data will get appended to the end of the file.
(*)
# Bash has no shorthand syntax that allows piping only StdErr to a second command, which would be needed here in combination with tee again to complete the table. If you really need something like that, please look at "How to pipe stderr, and not stdout?" on Stack Overflow for some ways how this can be done e.g. by swapping streams or using process substitution.
command |& tee output.txt
# Both the standard output and standard error streams will be copied to the file while still being visible in the terminal. If the file already exists, it gets overwritten.
command |& tee -a output.txt
# Both the standard output and standard error streams will be copied to the file while still being visible in the terminal. If the file already exists, the new data will get appended to the end of the file.
### Removing files
# To remove a folder with all its contents (including all interior folders):
rm -rf /path/to/directory
# To remove all the contents of the folder (including all interior folders) but not the folder itself:
rm -rf /path/to/directory/*
# or, if you want to make sure that hidden files/directories are also removed:
rm -rf /path/to/directory/{*,.*}
# To remove all the "files" from inside a folder(not removing interior folders):
rm -f /path/to/directory/{*,.*}
## Delete all file except file1 ##
rm !(file1)
## Delete all file except file1 and file2 ##
rm !(file1|file2)
## Delete all file except all zip files ##
rm !(*.zip)
## Delete all file except all zip and iso files using '*' wildcard ##
rm !(*.zip|*.iso)
## You set full path too ##
rm /Users/ffi007/!(*.zip|*.iso|*.mp3)
## Pass options to the rm ##
rm [options] !(*.zip|*.iso)
rm -v !(*.zip|*.iso) #verbose option
rm -f !(*.zip|*.iso) #force option
rm -v -i !(*.php) #confirm and verbose option
for x in *L6_Aligned.sortedByCoord.out.bam
do
mv "$x" "${x%_Aligned.sortedByCoord.out.bam}"
done
# To download FASTA files from NCBI. Go to page, view source, copy 'ncbi_uidlist', and paste to query string.
# Use
wget -O gata3.fq \
'https://www.ncbi.nlm.nih.gov/sviewer/viewer.fcgi?id=180038037&db=nuccore&report=fasta&retmode=text&withmarkup=on&tool=portal&log$=seqview&maxdownloadsize=100000000'