Geek Answers Handbook

What is the average ratio between the data file and the ETS table?

I appreciate using Erlang ETS to store a large dataset in memory, which. My test data source is a CSV file that consumes only 350 megabytes of disk.

My parser reads line by line and inserts it into a list, then creates a tuple and saves it in ETS using the bag configuration.

After loading all the data into the ETS, I noticed that my computer 8 GB of RAM disappeared, and the OS created a virtual memory that occupies about 16 GB or RAM. The erlang Beam process seems to consume about 10 times more memory than the size of the data on disk.

Here is the test code :

-module(load_test_data).
-author("gextra").

%% API
-export([test/0]).

init_ets() ->
  ets:new(memdatabase, [bag, named_table]).

parse(File) ->
  {ok, F} = file:open(File, [read, raw]),
  parse(F, file:read_line(F), []).

parse(F, eof, Done) ->
  file:close(F),
  lists:reverse(Done);

parse(F, Line, Done) ->
  parse(F, file:read_line(F), [ parse_row_commodity_data(Line) | Done ]).

parse_row_commodity_data(Line) ->
  {ok, Data} = Line,
  %%io:fwrite(Data),
  LineList          = re:split(Data,"\,",[{return,list}]),
  ReportingCountry  = lists:nth(1, LineList),
  YearPeriod        = lists:nth(2, LineList),
  Year              = lists:nth(3, LineList),
  Period            = lists:nth(4, LineList),
  TradeFlow         = lists:nth(5, LineList), 
  Commodity         = lists:nth(6, LineList),
  PartnerCountry    = lists:nth(7, LineList),
  NetWeight         = lists:nth(8, LineList),
  Value             = lists:nth(9, LineList),
  IsReported        = lists:nth(10, LineList),
  ets:insert(memdatabase, {YearPeriod ++ ReportingCountry ++ Commodity , { ReportingCountry, Year, Period, TradeFlow, Commodity, PartnerCountry, NetWeight, Value, IsReported } }).


test() ->
  init_ets(),
  parse("/data/000-2010-1.csv").
+4
1

, , , . . 16B, . 5.6GB .

:

parse(File) ->
  {ok, F} = file:open(File, [read, raw, binary]),
  ok = parse(F, binary:compile_pattern([<<$,>>, <<$\n>>])),
  ok = file:close(F).

parse(F, CP) ->
  case file:read_line(F) of
    {ok, Line} ->
      parse_row_commodity_data(Line, CP),
      parse(F, CP);
    eof -> ok
  end.

parse_row_commodity_data(Line, CP) ->
  [ ReportingCountry, YearPeriod, Year, Period, TradeFlow, Commodity,
    PartnerCountry, NetWeight, Value, IsReported]
      = binary:split(Line, CP, [global, trim]),
  true = ets:insert(memdatabase, {
         {YearPeriod, ReportingCountry, Commodity},
         { ReportingCountry, Year, Period, TradeFlow, Commodity,
           PartnerCountry, NetWeight, Value, IsReported}
       }).
+4

More articles:


All Articles