How can I combine multiple collections into one collection using $ lookup mongodb or nodejs mongodb?

Question

I have collections with the number of documents division - 30,

category - 248

productgroup - 1402

product - 60,000

For example, I have given data below

{ "divisionid": "1", "divisioncode": "0", "divisionname": "ELECTRONICS/HOME APPLIANCE", "divisionpoint": "2" }, { "divisionid": "2", "divisioncode": "1", "divisionname": "FOODS", "divisionpoint": "8" } 

collection data collection

 { "categoryid": "1", "divisionid": "1", "categorycode": "34", "categoryname": "AUDIO SYSTEM", "categorypoint": "Null" }, { "categoryid": "2", "divisionid": "1", "categorycode": "348", "categoryname": "DVD/VCD", "categorypoint": "8" } 

product collection collection data

 { "productgroupid": "1", "divisionid": "1", "categoryid": "1", "productgroupname": "ADAPTOR", "productgroupcode": "6765", "productgrouppoint": "7" }, { "productgroupid": "2", "divisionid": "1", "categoryid": "2", "productgroupname": "WALKMAN", "productgroupcode": "7659", "productgrouppoint": "Null" } 

data collection data

 { "productid": "1", "divisionid": "1", "categoryid": "1", "productgroupid":"1", "productname":"UNIVERSAL AC DC ADAPTER-PCS", "productcode": "1000054", "productpoint": "1" }, { "productid": "2", "divisionid": "1", "categoryid": "2", "productgroupid":"2", "productname":"WALKMAN WM#M470-PCS", "productcode": "1000089", "productpoint": "2" } 

I have 4 collections like

division,

category,

Product group

product

all this is my data that is stored in the data collection of my collection data

{

 "divisionid": "1", "divisioncode": "0", "divisionname": "ELECTRONICS/HOME APPLIANCE", "divisionpoint": "2" 

}, {

 "divisionid": "2", "divisioncode": "1", "divisionname": "FOODS", "divisionpoint": "8" 

} collection collection data

{

 "categoryid": "1", "divisionid": "1", "categorycode": "34", "categoryname": "AUDIO SYSTEM", "categorypoint": "Null" 

}, {

 "categoryid": "2", "divisionid": "1", "categorycode": "348", "categoryname": "DVD/VCD", "categorypoint": "8" 

}

{

 "productgroupid": "1", "divisionid": "1", "categoryid": "1", "productgroupname": "ADAPTOR", "productgroupcode": "6765", "productgrouppoint": "7" 

}, {

 "productgroupid": "2", "divisionid": "1", "categoryid": "2", "productgroupname": "WALKMAN", "productgroupcode": "7659", "productgrouppoint": "Null" 

} data collection data

{

 "productid": "1", "divisionid": "1", "categoryid": "1", "productgroupid":"1", "productname":"UNIVERSAL AC DC ADAPTER-PCS", "productcode": "1000054", "productpoint": "1" 

}, {

 "productid": "2", "divisionid": "1", "categoryid": "2", "productgroupid":"2", "productname":"WALKMAN WM#M470-PCS", "productcode": "1000089", "productpoint": "2" 

} I want to combine these 4 collections into one collection. should i get a resume like this productsummary

 { "productpoint": "1", "productname": "UNIVERSAL AC DC ADAPTER-PCS", "productcode": "10000054", "productid" :"1" "group": { "point": "7", "name": "ADAPTOR", "id" :"1" }, "category": { "point": "0", "name": "AUDIO SYSTEM", "id" :"1" }, "division": { "point": "2", "name": "ELECTRONICS/HOME APPLIANCE", "id" :"1" } }, { "productpoint": "2", "productname": "WALKMAN WM#M470-PCS", "productcode": "1000089", "productid" :"2" "group": { "point": "7", "name": "WALKMAN", "id" :"Null" }, "category": { "point": "8", "name": "DVD/VCD", "id" :"2" }, "division": { "point": "2", "name": "ELECTRONICS/HOME APPLIANCE", "id" :"1" } } 

i run this code

 db.products.aggregate([ {$lookup:{from:"productgroups", localField:"productgroupid", foreignField:"productgroupid", as:"group"}}, {$lookup:{from:"category", localField:"categoryid", foreignField:"categoryid", as:"category"}}, {$lookup:{from:"divisions", localField:"divisionid", foreignField:"divisionid", as:"division"}}, {$project: {productpoint:1,productname:1,productcode:1,productid:1,group: { $arrayElemAt: [ "$group", 0 ]}, category: { $arrayElemAt: [ "$category", 0 ]}, division: { $arrayElemAt: [ "$division", 0 ]} }} ]); 

its work, but I had 60 k products that its time did not work and got an error similar to this

and the data is stored in the same way as in the product points of memory

 { "productpoint": "2", "productname": "WALKMAN WM#M470-PCS", "productcode": "1000089", "productid" :"2" } 

is there any other way to do this and give some other solutions for this

is there another option in nodejs mongodb?

Answer 1

You must use $ skip and $ limit to avoid this error. why get this error? you should read the @Alex comment link.

 db.products.aggregate([ {$skip:0},{$limit: 1000}, {$lookup:{from:"productgroups", localField:"productgroupid", foreignField:"productgroupid", as:"group"}}, {$lookup:{from:"category", localField:"categoryid", foreignField:"categoryid", as:"category"}}, {$lookup:{from:"divisions", localField:"divisionid", foreignField:"divisionid", as:"division"}}, {$project: {productpoint:1,productname:1,productcode:1,productid:1,group: { $arrayElemAt: [ "$group", 0 ]}, category: { $arrayElemAt: [ "$category", 0 ]}, division: { $arrayElemAt: [ "$division", 0 ]} }} ]); 

{$skip:0},{$limit: 1000} // for the initial one, and then for the next time should skip: 1000, limit: 1000 and so on. you can get it from the query either with a loop or with its implementation.