SED or AWK replace all templates from another file

Question

SED or AWK replace all templates from another file

I am trying to do a template replacement using a SED script, but it does not work properly

sample_content.txt

288Y2RZDBPX1000000001dhana
JP2F64EI1000000002d
EU9V3IXI1000000003dfg1000000001dfdfds
XATSSSSFOO4dhanaUXIBB7TF71000000004adf
10Q1W4ZEAV18LXNPSPGRTTIDHBN1000000005egw

patterns.txt

1000000001 9000000003
1000000002 2000000001
1000000003 3000000001
1000000004 4000000001
1000000005 5000000001

Expected Result

288Y2RZDBPX9000000003dhana
JP2F64EI2000000001d
EU9V3IXI3000000001dfg9000000003dfdfds
XATSSSSFOO4dhanaUXIBB7TF74000000001adf
10Q1W4ZEAV18LXNPSPGRTTIDHBN5000000001egw

I can do with one replacement SED, for example,

sed  's/1000000001/1000000003/g' sample_content.txt

Note:

The corresponding template is not in a fixed position.
A single line can have multiple matching values to replace in sample_content.txt
Sample_content.txt and patterns.txt has> 1 million entries

File Link: https://drive.google.com/open?id=1dVzivKMirEQU3yk9KfPM6iE7tTzVRdt_

Can anyone suggest how to achieve this without affecting performance?

Updated February 11, 2018

, 30 31- . , .
AB, 10- 41-50 101-110
BC, 10- 11-20, 61-70 151-160
DE, 10- 1-10, 71-80, 151-160 181-190

50 2 ..

{   grade=substr($0,110,2)} // identify grade
{ 
    if (grade == "AB") {
        print substr($0,41,10) ORS substr($0,101,10)
    } else if(RT == "BC"){
        print substr($0,11,10) ORS substr($0,61,10) ORS substr($0,151,10) 
    }

    like wise 50 coiditions
}

, - ?

+6

shell awk perl sed gawk

Dhanabalan 10 . '18 9:41

4

awk , .

1st:

awk 'FNR==NR{a[$1]=$2;next}   {for(i in a){match($0,i);val=substr($0,RSTART,RLENGTH);if(val){sub(val,a[i])}};print}' patterns.txt  sample_content.txt

.

288Y2RZDBPX9000000003dhana
JP2F64EI2000000001d
EU9V3IXI3000000001dfg9000000003dfdfds
XATSSSSFOO4dhanaUXIBB7TF74000000001adf
10Q1W4ZEAV18LXNPSPGRTTIDHBN5000000001egw

1st: :

awk '
FNR==NR{                           ##FNR==NR is a condition which will be TRUE when only first Input_file patterns.txt is being read.
                                   ##FNR and NR both represents line number of Input_file(s) where FNR value will be RESET when a new Input_file is getting read on the other hand NR value will be keep increasing till all Input_file(s) read.
  a[$1]=$2;                        ##creating an array a whose index is first field of line and value is 2nd field of current line.
  next                             ##next will skip all further statements for now.
}
{
for(i in a){                       ##Starting a for loop which traverse through array a all element.
  match($0,i);                     ##Using match function of awk which will try to match index if array a present in variable i.
  val=substr($0,RSTART,RLENGTH);   ##Creating a variable named val which contains the substring of current line substring starts from value of variable RSTART till RLENGTH value.
  if(val){                         ##Checking condition if variable val is NOT NULL then do following:
    sub(val,a[i])}                 ##using sub function of awk to substitute variable val value with array a value of index i.
};
  print                            ##Using print here to print the current line either changed or not changed one.
}
' patterns.txt  sample_content.txt ##Mentioning the Input_file(s) name here.

2nd: , , , :

awk '
FNR==NR{                           ##FNR==NR is a condition which will be TRUE when only first Input_file patterns.txt is being read.
                                   ##FNR and NR both represents line number of Input_file(s) where FNR value will be RESET when a new Input_file is getting read on the other hand NR value will be keep increasing till all Input_file(s) read.
  a[$1]=$2;                        ##creating an array a whose index is first field of line and value is 2nd field of current line.
  next                             ##next will skip all further statements for now.
}
{
for(i in a){                       ##Starting a for loop which traverse through array a all element.
  match($0,i);                     ##Using match function of awk which will try to match index if array a present in variable i.
  val=substr($0,RSTART,RLENGTH);   ##Creating a variable named val which contains the substring of current line substring starts from value of variable RSTART till RLENGTH value.
  if(val){                         ##Checking condition if variable val is NOT NULL then do following:
    sub(val,a[i]);print;next}                 ##using sub function of awk to subsitute variable val value with array a value of index i.
};
}
1
' patterns.txt  sample_content.txt ##Mentioning the Input_file(s) name here.

+3

RavinderSingh13 10 . '18 9:49

. .

$ sed -f <(printf 's/%s/%s/g\n' $(<patterns.txt)) contents.txt

`patterns.txt, , patterns.txt:

$ printf 's/%s/%s/g\n' $(<patterns.txt)
s/1000000001/9000000003/g
s/1000000002/2000000001/g
s/1000000003/3000000001/g
s/1000000004/4000000001/g
s/1000000005/5000000001/g

<(...) sed script, sed -f switch = sed

$ sed -f <(printf 's/%s/%s/g\n' $(<patterns.txt)) contents.txt
288Y2RZDBPX9000000003dhana
JP2F64EI2000000001d
EU9V3IXI3000000001dfg9000000003dfdfds
XATSSSSFOO4dhanaUXIBB7TF74000000001adf
10Q1W4ZEAV18LXNPSPGRTTIDHBN5000000001egw

+3

George Vasiliou 10 . '18 10:07

:

$ cat tst.awk
NR==FNR {
    map[$1] = $2
    next
}
{
    for (old in map) {
        gsub(old,map[old])
    }
    print
}

$ awk -f tst.awk patterns.txt sample_content.txt
288Y2RZDBPX9000000003dhana
JP2F64EI2000000001d
EU9V3IXI3000000001dfg9000000003dfdfds
XATSSSSFOO4dhanaUXIBB7TF74000000001adf
10Q1W4ZEAV18LXNPSPGRTTIDHBN5000000001egw

, , , sample_content.txt, xay patterns.txt, a b b c, xcy xby.

:

$ cat tst.awk
NR==FNR {
    map[$1] = $2
    re = re sep $1
    sep = "|"
    next
}
{
    head = ""
    tail = $0
    while ( match(tail,re) ) {
        head = head substr(tail,1,RSTART-1) map[substr(tail,RSTART,RLENGTH)]
        tail = substr(tail,RSTART+RLENGTH)
    }
    print head tail
}

$ awk -f tst.awk patterns.txt sample_content.txt
288Y2RZDBPX9000000003dhana
JP2F64EI2000000001d
EU9V3IXI3000000001dfg9000000003dfdfds
XATSSSSFOO4dhanaUXIBB7TF74000000001adf
10Q1W4ZEAV18LXNPSPGRTTIDHBN5000000001egw

:

xby ( , , ) ,
sample_content.txt, , 1 patterns.txt sample_content.txt
, , .
It does not change $0, and therefore awk does not need to recompile and overwrite this entry with each subscription.

therefore, it should be much faster than the original script, assuming that the regular expression built from patterns.txt is not so large, it leads to performance degradation only in size.

+2

Ed morton Feb 10 '18 at 11:45

source share

George Vasiliou · Accepted Answer · 2018-02-10T16:33:49+0000

:

patterns.txt 50 000 contents.txt 50 000 .

patterns.txt , 1000 contents.txt.

64- Intel (R) Celeron (R) N3050 @2,16 , 4 , Debian 9 64bit Testing, gnu sed 4.4 gnu awk 4.1.4

, .

:

1. RavinderSingh13 1- awk

$ time awk 'FNR==NR{a[$1]=$2;next}   {for(i in a){match($0,i);val=substr($0,RSTART,RLENGTH);if(val){sub(val,a[i])}};print}' patterns.txt  <(head -n 1000 contents.txt) >newcontents.txt

real    19m54.408s
user    19m44.097s
sys 0m1.981s

2. Edmorton 1st awk

$ time awk 'NR==FNR{map[$1]=$2;next}{for (old in map) {gsub(old,map[old])}print}' patterns.txt <(head -n1000 contents.txt) >newcontents.txt

real    20m3.420s
user    19m16.559s
sys 0m2.325s

3. Sed ( sed)

$ time sed -f <(printf 's/%s/%s/g\n' $(<patterns.txt)) <(head -n 1000 contents.txt) >newcontents.txt

real    1m1.070s
user    0m59.562s
sys 0m1.443s

4. Cyrus sed

$ time sed -f <(sed -E 's|(.*) (.*)|s/\1/\2/|g' patterns.txt) <(head -n1000 contents.txt) >newcontents.txt

real    1m0.506s
user    0m59.871s
sys 0m1.209s

5. RavinderSingh13 2- awk

$ time awk 'FNR==NR{a[$1]=$2;next}{for(i in a){match($0,i);val=substr($0,RSTART,RLENGTH);if(val){sub(val,a[i]);print;next}};}1' patterns.txt  <(head -n 1000 contents.txt) >newcontents.txt

real    0m25.572s
user    0m25.204s
sys     0m0.040s

, 1000 , awk- . 9000 ,

6.RavinderSingh13 2- awk 9000

$ time awk 'FNR==NR{a[$1]=$2;next}{for(i in a){match($0,i);val=substr($0,RSTART,RLENGTH);if(val){sub(val,a[i]);print;next}};}1' patterns.txt  <(head -9000 contents.txt) >newcontents.txt

real    22m25.222s
user    22m19.567s
sys      0m2.091s

7. Sed Solution 9000

$ time sed -f <(printf 's/%s/%s/g\n' $(<patterns.txt)) <(head -9000 contents.txt) >newcontents.txt

real    9m7.443s
user    9m0.552s
sys     0m2.650s

8. 9000

$ cat sedpar.sh
s=$SECONDS
sed -f <(printf 's/%s/%s/g\n' $(<patterns.txt)) <(head -3000 contents.txt) >newcontents1.txt &
sed -f <(printf 's/%s/%s/g\n' $(<patterns.txt)) <(tail +3001 contents.txt |head -3000) >newcontents2.txt &
sed -f <(printf 's/%s/%s/g\n' $(<patterns.txt)) <(tail +6001 contents.txt |head -3000) >newcontents3.txt &
wait
cat newcontents1.txt newcontents2.txt newcontents3.txt >newcontents.txt && rm -f newcontents1.txt newcontents2.txt newcontents3.txt
echo "seconds elapsed: $(($SECONDS-$s))"

$ time ./sedpar.sh
seconds elapsed: 309

real    5m16.594s
user    9m43.331s
sys     0m4.232s

, , , .

, , contents.txt patterns.txt OP github:

contents.txt

patterns.txt

SED or AWK replace all templates from another file

More articles: