#################
# Count with wc #
#################

# number of lines
wc -l adult.data
# number of words
wc -w adult.data
# number of characters
wc -m adult.data

########################
# Concatenate with cat #
########################

# there is no header
head -n 2 adult.data

# create header
echo "age,workclass,fnlwgt,education,education_num,marital_status,occupation,relationship,race,sex,capital_gain,capital_loss,hours_per_week,native_country,class" > header.csv

# add header and rename
cat header.csv adult.data > adult.csv

# check
head -n 2 adult.csv

##########################
# Modify a file with sed #
##########################

# grep (select lines containing) missing values (represented with ?)
grep ", ?," adult.csv | head -n 3

# count lines containing a missing value
grep ", ?," adult.csv | wc -l

# replace missing values with empty string
# syntax: s/regexp/replacement/flag
sed "s/, ?,/,,/g" adult.csv >  adult2.csv

# rename adult2.csv as adult.csv
mv adult2.csv adult.csv

# check
grep ", ?," adult.csv | wc -l

#######################
# Subset a large file #
#######################

# first n lines
head -n 10 adult.csv

# last n lines
tail -n 10 adult.csv

# 20 lines starting at line 100
head -n 120 adult.csv | tail -n 20 > adult_sample.csv

# add header to sample
cat header.csv adult_sample.csv  > adult_sample_with_header.csv

################################
# Finding duplicates with uniq #
################################

# sort lines
sort adult.csv | head 

# count duplicated lines
sort adult.csv | uniq -d | wc -l

# find most repeated lines
sort adult.csv | uniq -c | sort -r | head 

# remove duplicate rows
sort adult.csv | uniq -u > adult_unique.csv
# check
uniq -d adult_unique.csv | wc -l

##############################
# Selecting columns with cut #
##############################

# select second column (workclass)
cut -d "," -f 2 adult.csv | head -3


# count unique workclasses
cut -d "," -f 2 adult.csv | sort | uniq -c