- create a file with duplicate records
1
2
3
4
5
6
7
8
9
10
11
$ echo "test" > space.txt
$ echo " test" >> space.txt
$ echo " test " >> space.txt
$ echo "test " >> space.txt
$ echo "test1" >> space.txt
$ echo "test2" >> space.txt
- list records
1
2
3
4
5
6
7
8
$ cat space.txt
test
test
test
test
test1
test2
- remove trailing white space using awk
1
2
3
4
5
6
7
8
$ cat space.txt | awk '{$1=$1;print}'
test
test
test
test
test1
test2
- sort and then get all unique records
1
2
3
4
5
$ cat space.txt | awk '{$1=$1;print}' | sort | uniq
test
test1
test2