Related
I have a large collection about 200 mil documents. Documents are small but if I try to aggregate some data it fall down on timeout error. I am not sure if it is caused by indexes or by documents count.
The pipeline looks like:
{
$match: {
global_campaign_id: {$in: [3151 ,3207 ,3208 ,3209 ,3210 , ... (30 ids)]}
}
},
{
$group: {
_id: {
global_campaign_id: "$global_campaign_id",
device_id: "$device_id",
partner_id: "$partner_id"
},
date_last: {
$max: "$date_created"
},
partner_id: {$first: "$partner_id"},
campaign_id: {$first: "$global_campaign_id"},
device_id: {$first: "$device_id"}
}
}
Then I have this idexes on the collection:
_id: 1
date_created: 1,
global_campaign_id: 1,
global_campaign_id: 1, date_created: 1,
global_campaign_id: 1, device_id: 1, partner_id: 1, date_created: 1
What could be the problem? I need last document for every group. Could sort stage before group help?
Here is result of explain command
{
"stages" : [
{
"$cursor" : {
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "lurity.impressions",
"indexFilterSet" : false,
"parsedQuery" : {
"global_campaign_id" : {
"$in" : [
3151,
3207,
3208,
3209,
3210
]
}
},
"queryHash" : "901B446C",
"planCacheKey" : "06CC82FF",
"winningPlan" : {
"stage" : "PROJECTION_COVERED",
"transformBy" : {
"date_created" : 1,
"device_id" : 1,
"global_campaign_id" : 1,
"partner_id" : 1,
"_id" : 0
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"global_campaign_id" : 1,
"device_id" : 1,
"partner_id" : 1,
"date_created" : 1
},
"indexName" : "global_campaign_id_1_device_id_1_partner_id_1_date_created_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"global_campaign_id" : [ ],
"device_id" : [ ],
"partner_id" : [ ],
"date_created" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"global_campaign_id" : [
"[3151.0, 3151.0]",
"[3207.0, 3207.0]",
"[3208.0, 3208.0]",
"[3209.0, 3209.0]",
"[3210.0, 3210.0]"
],
"device_id" : [
"[MinKey, MaxKey]"
],
"partner_id" : [
"[MinKey, MaxKey]"
],
"date_created" : [
"[MinKey, MaxKey]"
]
}
}
},
"rejectedPlans" : [
{
"stage" : "PROJECTION_SIMPLE",
"transformBy" : {
"date_created" : 1,
"device_id" : 1,
"global_campaign_id" : 1,
"partner_id" : 1,
"_id" : 0
},
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"global_campaign_id" : 1
},
"indexName" : "global_campaign_id_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"global_campaign_id" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"global_campaign_id" : [
"[3151.0, 3151.0]",
"[3207.0, 3207.0]",
"[3208.0, 3208.0]",
"[3209.0, 3209.0]",
"[3210.0, 3210.0]"
]
}
}
}
},
{
"stage" : "PROJECTION_SIMPLE",
"transformBy" : {
"date_created" : 1,
"device_id" : 1,
"global_campaign_id" : 1,
"partner_id" : 1,
"_id" : 0
},
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"global_campaign_id" : 1,
"date_created" : 1
},
"indexName" : "global_campaign_id_1_date_created_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"global_campaign_id" : [ ],
"date_created" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"global_campaign_id" : [
"[3151.0, 3151.0]",
"[3207.0, 3207.0]",
"[3208.0, 3208.0]",
"[3209.0, 3209.0]",
"[3210.0, 3210.0]"
],
"date_created" : [
"[MinKey, MaxKey]"
]
}
}
}
}
]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 304244,
"executionTimeMillis" : 1363,
"totalKeysExamined" : 304246,
"totalDocsExamined" : 0,
"executionStages" : {
"stage" : "PROJECTION_COVERED",
"nReturned" : 304244,
"executionTimeMillisEstimate" : 112,
"works" : 304246,
"advanced" : 304244,
"needTime" : 1,
"needYield" : 0,
"saveState" : 322,
"restoreState" : 322,
"isEOF" : 1,
"transformBy" : {
"date_created" : 1,
"device_id" : 1,
"global_campaign_id" : 1,
"partner_id" : 1,
"_id" : 0
},
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 304244,
"executionTimeMillisEstimate" : 73,
"works" : 304246,
"advanced" : 304244,
"needTime" : 1,
"needYield" : 0,
"saveState" : 322,
"restoreState" : 322,
"isEOF" : 1,
"keyPattern" : {
"global_campaign_id" : 1,
"device_id" : 1,
"partner_id" : 1,
"date_created" : 1
},
"indexName" : "global_campaign_id_1_device_id_1_partner_id_1_date_created_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"global_campaign_id" : [ ],
"device_id" : [ ],
"partner_id" : [ ],
"date_created" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"global_campaign_id" : [
"[3151.0, 3151.0]",
"[3207.0, 3207.0]",
"[3208.0, 3208.0]",
"[3209.0, 3209.0]",
"[3210.0, 3210.0]"
],
"device_id" : [
"[MinKey, MaxKey]"
],
"partner_id" : [
"[MinKey, MaxKey]"
],
"date_created" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 304246,
"seeks" : 2,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
}
},
"nReturned" : NumberLong(304244),
"executionTimeMillisEstimate" : NumberLong(803)
},
{
"$group" : {
"_id" : {
"global_campaign_id" : "$global_campaign_id",
"device_id" : "$device_id",
"partner_id" : "$partner_id"
},
"date_last" : {
"$max" : "$date_created"
},
"partner_id" : {
"$first" : "$partner_id"
},
"campaign_id" : {
"$first" : "$global_campaign_id"
},
"device_id" : {
"$first" : "$device_id"
}
},
"nReturned" : NumberLong(84),
"executionTimeMillisEstimate" : NumberLong(1358)
}
],
"serverInfo" : {
"host" : "nosql-lurity",
"port" : 27017,
"version" : "4.4.13",
"gitVersion" : "df25c71b8674a78e17468f48bcda5285decb9246"
},
"ok" : 1
}
i have got a mongo array , in which i want to remove the whole block , if the nested array of that block is empty
I have attached the array below
{
"_id" : ObjectId("5b17b991c440782b5a218cd1"),
"vendor_view_id" : 741733,
"product" : [
{
"id" : ObjectId("5b86546540c1c414543e4333"),
"vendor_user_id" : ObjectId("5b17b992c440782b5a218cd2"),
"product_type_id" : ObjectId("5ae8348b7ae0d9538e45ab46"),
"condition_id" : [ ],
"shipping_cost" : 100,
"date_added" : "2018-08-29-08-08-05",
"date_status_change" : "2018-08-29-08-08-05",
"status" : 0
},
{
"id" : ObjectId("5b8654ba40c1c4145d1f5473"),
"vendor_user_id" : ObjectId("5b17b992c440782b5a218cd2"),
"product_type_id" : ObjectId("5ae834b17ae0d9538e45ab48"),
"condition_id" : [ ],
"shipping_cost" : 100,
"date_added" : "2018-08-29-08-09-30",
"date_status_change" : "2018-08-29-08-09-30",
"status" : 0
},
{
"id" : ObjectId("5b8655a840c1c415080b0a33"),
"vendor_user_id" : ObjectId("5b17b992c440782b5a218cd2"),
"product_type_id" : ObjectId("5ae834a67ae0d9538e45ab47"),
"condition_id" : [
{
"_id" : ObjectId("5ae977da7ff1706f3b7dc47a"),
"status" : 0,
"date_added" : "2018-08-29-08-13-28"
}
],
"shipping_cost" : 100,
"date_added" : "2018-08-29-08-13-28",
"date_status_change" : "2018-08-29-08-13-28",
"status" : 0
}
]
}
I would like to delete the array block where product.condition_id is empty
So far i have tried this
$this->collection_name->collection->updateOne([
'_id' => $vendor_id,
],
[
'$unset' =>
[
'product.$.condition_id' =>
[
'$size'=>0,
]
]
])
EDIT 1:
db.collection_name.collection({_id : ObjectId('5b17b991c440782b5a218cd1'),
"product.condition_id.$":{ "$exists": false }},
{ "$unset": { "product.$": "" }});
still not working
db.collection_name.update(
{},
{$pull : {product : {condition_id : {$size : 0} }}},
{ multi: true } // multi : true will updates multiple documents that meet the query criteria
)
if condition_id array is empty pull the document.
Output:
{
"_id" : ObjectId("5b17b991c440782b5a218cd1"),
"vendor_view_id" : 741733,
"product" : [
{
"id" : ObjectId("5b8655a840c1c415080b0a33"),
"vendor_user_id" : ObjectId("5b17b992c440782b5a218cd2"),
"product_type_id" : ObjectId("5ae834a67ae0d9538e45ab47"),
"condition_id" : [
{
"_id" : ObjectId("5ae977da7ff1706f3b7dc47a"),
"status" : 0,
"date_added" : "2018-08-29-08-13-28"
}
],
"shipping_cost" : 100,
"date_added" : "2018-08-29-08-13-28",
"date_status_change" : "2018-08-29-08-13-28",
"status" : 0
}
]
}
You can transform your json to PHP array json_decode($json) so you'll be able to process it before it turn into json after.
json_encode($arr)
Note that you can check the depth of the array in your case to see if product.condition_id is empty.
Hi all i am trying to push values to existing record using $push but i am getting error stating that
Invalid modifier specified: $push
I am using php below is my code php code
$collection->update(array('_id'=>3,"data._id"=>2),array('$push'=>array('userid','52')));
i.e adding 52 to userid. In 3 record and data._id 2
below is my table structure for mongo db
{ "_id" : 2,
"name" : "test",
"data" :[{"_id" : "1",
"file" : "nic",
"userid" : [1,2 ]
},
{"_id" : "2",
"file" : "nic1",
"userid" : [1 ]
},
{"_id" : 3,
"file" : "nick2",
"userid" : [1,2 ]
}
]},
{ "_id" : 3,
"name" : "test",
"data" : [{"_id" : "1",
"file" : "nic",
"userid" : [1,2 ]
},
{"_id" : "2",
"file" : "nic1",
"userid" : [3,2 ]
}
]}
Use the $ positional operator in your update that identifies an element in the data array to update without explicitly specifying the position of the element:
$collection -> update(
array('_id' => 3, "data._id" => 2),
array('$push' =>
array('data.$.userid' => 52)
)
);
This question already has an answer here:
Places Autocomplete API to get GPS coordinates from address entered
(1 answer)
Closed 8 years ago.
Suppose a very simple function to query Query Autocomplete:
$query_google_places = function ($input, array $location = null) {
$parameters = ['key' => 'AIzaSyAjCY7qYiaMv8Kkk_-D1Ha7wlRkr5neppk', 'input' => $input, 'sensor' => 'false'];
if ($location) {
$parameters['location'] = $location['lat'] . ',' . $location['lng'];
$parameters['radius'] = 10000;
}
$url = 'https://maps.googleapis.com/maps/api/place/queryautocomplete/json?' . \http_build_query($parameters);
$response = file_get_contents($url);
return $response;
};
This returns:
{
"predictions" : [
{
"description" : "Pizza Hut, Halifax, United Kingdom",
"id" : "d9274717e1e203c41287fa5936701b9584814965",
"matched_substrings" : [
{
"length" : 9,
"offset" : 0
}
],
"reference" : "CmRaAAAAyulRkdspSJ2Mr5kmlhYOC-ZT7AsKdnnwjGkFFoGHNjMWwTS5mNvQBu6FqoQ89E4U0eC-PNGN0wPD8WEYgO1if6jMmxWAqkraDFOryC8B8cWTvX333l_0UaRv_sXBIITqEhCSrz9aQB0fch-AFDS3wKG7GhTRoQw09IODWJIorqjjwCYTktkfKw",
"terms" : [
{
"offset" : 0,
"value" : "Pizza Hut"
},
{
"offset" : 11,
"value" : "Halifax"
},
{
"offset" : 20,
"value" : "United Kingdom"
}
],
"types" : [ "establishment" ]
},
{
"description" : "Pizza Hut, John William Street, Huddersfield, United Kingdom",
"id" : "a1571b1434ac1491b5f6775d9a30f8ec7798a310",
"matched_substrings" : [
{
"length" : 9,
"offset" : 0
}
],
"reference" : "CoQBdAAAADaFqqtB9NGk_v1yZdY3m6OxILfLva556GBHxHIVdPHu_R0fQCm5kCduCjKL4BgiIdoNIejet3WI7xNanaiWDpiMD1Ml7Q9EnfhypReK0N6mQIY3TxOUcmf_INhkenU3ZVL8Vk9lAxqjq5ZfxiEZk9Vv8GaEnok_OJnaRBuZdCmtEhAYs-0Qc2Zza4fLuUXIiLknGhQogPaBcUm10Y59EA5i6BWPbTNXcA",
"terms" : [
{
"offset" : 0,
"value" : "Pizza Hut"
},
{
"offset" : 11,
"value" : "John William Street"
},
{
"offset" : 32,
"value" : "Huddersfield"
},
{
"offset" : 46,
"value" : "United Kingdom"
}
],
"types" : [ "establishment" ]
},
{
"description" : "pizza hut",
"matched_substrings" : [
{
"length" : 9,
"offset" : 0
}
],
"terms" : [
{
"offset" : 0,
"value" : "pizza hut"
}
]
},
{
"description" : "Pizza Hut Delivery, Wakefield Road, Huddersfield, United Kingdom",
"id" : "0c6830de14a8e23bf971626e880201f5290ec9a6",
"matched_substrings" : [
{
"length" : 9,
"offset" : 0
}
],
"reference" : "CoQBeQAAAJnMKHhJh-IE64z6seh3sbnm4dt0h1zbMYLr4BPGCITQTZLp4lkm12HKEFw0Bt761UaZPsESrY4ha7XSXusqqRVstmRFBF2wiwF5HGMO9DhRSeIzJ8CkNvsxmlqINosGBBPUHEDQEeSOoPHn3u3MeByoLVw6AzZ9N6eTrm94hsX9EhBB9cIoXjhHjR9Tr3zBROM7GhRhpYagW9qWs9hGZ3V0o9WXfrfcXw",
"terms" : [
{
"offset" : 0,
"value" : "Pizza Hut Delivery"
},
{
"offset" : 20,
"value" : "Wakefield Road"
},
{
"offset" : 36,
"value" : "Huddersfield"
},
{
"offset" : 50,
"value" : "United Kingdom"
}
],
"types" : [ "establishment" ]
},
{
"description" : "Pizza Hut Delivery, Westgate, Halifax, United Kingdom",
"id" : "667c14a2cb5bb676bd02911e924db82fb5cbe6df",
"matched_substrings" : [
{
"length" : 9,
"offset" : 0
}
],
"reference" : "CnRtAAAAEtMYQ9N4Gmn8dpZqzHOxYGPCNP9QQ6M-gHjj2hJv2euH4gUVvDhmgazi6LcRjdYpUF41moGLr26IGc2vOPfvFg_kHTqTIYHHwiD84bu0PeEzmLzIzJw2IkGNpcW1o6weO86TW8zaUaMFZ2zNW-tynhIQFT8GvSn_q7DAoP3ytM14ExoUN3sShljpfaCWEUknvsfAjZJ_Ru4",
"terms" : [
{
"offset" : 0,
"value" : "Pizza Hut Delivery"
},
{
"offset" : 20,
"value" : "Westgate"
},
{
"offset" : 30,
"value" : "Halifax"
},
{
"offset" : 39,
"value" : "United Kingdom"
}
],
"types" : [ "establishment" ]
}
],
"status" : "OK"
}
How do I get coordinates for each result? Note, that this is server-side question.
IMO, autocompletion or query autocompletion results live in a different world than search results. Google assumes you're not supposed to need coordinates in the autocompletion list because this is not the data you will be presenting on your map or in any controller. The data you'll present will be the result of a true "search" request.
Conclusion: you're supposed to call again Google server on endpoints such as nearby search, text search or details to get the coordinates.
If you want to use details, if the user clicked on a prediction from autocomplete for example, you'll use reference attribute as a parameter of your http request to identify the clicked prediction.
How to search value in multidimensional array,
for example I want to search example keyword in the following data in mongodb
I used to fetch all data from command
>db.info.find()
{
"_id" : ObjectId("4f74737cc3a51043d26f4b90"),
"id" : "12345",
"info" : [
{
"sno" : 1,
"name" : "ABC",
"email" : "abc#example.com"
},
{
"sno" : 2,
"name" : "XYZ",
"email" : "xyz#example.com"
},
{
"sno" : 3,
"name" : "XYZ",
"email" : "xyz#demo.com"
},
{
"sno" : 4,
"name" : "ABC",
"email" : "abc#demo.com"
},
{
"sno" : 5,
"name" : "Rohan",
"email" : "rohan#example.com"
}
]
}
Now, to find data having example I used command
>db.info.find({"info.email":"example"})
and it gives
{
"_id" : ObjectId("4f74737cc3a51043d26f4b90"),
"id" : "12345",
"info" : [
{
"sno" : 1,
"name" : "ABC",
"email" : "abc#example.com"
},
{
"sno" : 2,
"name" : "XYZ",
"email" : "xyz#example.com"
},
{
"sno" : 3,
"name" : "XYZ",
"email" : "xyz#demo.com"
},
{
"sno" : 4,
"name" : "ABC",
"email" : "abc#demo.com"
},
{
"sno" : 5,
"name" : "Rohan",
"email" : "rohan#example.com"
}
]
}
But I want only 3 out of 5 sub rows like
{
"_id" : ObjectId("4f74737cc3a51043d26f4b90"),
"id" : "12345",
"info" : [
{
"sno" : 1,
"name" : "ABC",
"email" : "abc#example.com"
},
{
"sno" : 2,
"name" : "XYZ",
"email" : "xyz#example.com"
},
{
"sno" : 5,
"name" : "Rohan",
"email" : "rohan#example.com"
}
]
}
Rohan, MongoDB always returns the whole document that you are searching on. You can't just make it return the array elements in which your keyword was found. If you want to do that, then you need to make sure all all embedded documents in the "info" field are in their own collection. And that might mean that you need to link them back to the original document in your "info" collection. Perhaps something like:
{
"sno" : 1,
"name" : "ABC",
"email" : "abc#example.com"
"info_id" : "12345",
},
Alternatively, you can of course do post-processing in PHP to obtain only the rows that you want.
Perhaps this is a good idea?
http://php.net/manual/en/class.mongoregex.php
I tried Map Reduce Function and it works on this type of problems the code is something like that:
Write a map function
map=function ()
{
filter = [];
this.info.forEach(function (s) {if (/example/.test(s.email)) {filter.push(s);}});
emit(this._id, {info:filter});
}
Write a reduce function
reduce=function(key, values) { return values;}
MapReduce Function
res=db.info.mapReduce(map,reduce,{out:{inline:1}})
And The Output look likes:
"results" : [
{
"_id" : ObjectId("4f9a2de0ea4a65c3ab85a9d3"),
"value" : {
"info" : [
{
"sno" : 1,
"name" : "ABC",
"email" : "abc#example.com"
},
{
"sno" : 2,
"name" : "XYZ",
"email" : "xyz#example.com"
},
{
"sno" : 5,
"name" : "Rohan",
"email" : "rohan#example.com"
}
]
}
}
],
"timeMillis" : 1,
"counts" : {
"input" : 3,
"emit" : 3,
"reduce" : 0,
"output" : 3
},
"ok" : 1,
Now you can find your search data from
printjson(res.results)
Did you try $ (projection)?
db.info.find({"info.email":"example"}, {"info.email.$":1})
document