Related
I have a large collection about 200 mil documents. Documents are small but if I try to aggregate some data it fall down on timeout error. I am not sure if it is caused by indexes or by documents count.
The pipeline looks like:
{
$match: {
global_campaign_id: {$in: [3151 ,3207 ,3208 ,3209 ,3210 , ... (30 ids)]}
}
},
{
$group: {
_id: {
global_campaign_id: "$global_campaign_id",
device_id: "$device_id",
partner_id: "$partner_id"
},
date_last: {
$max: "$date_created"
},
partner_id: {$first: "$partner_id"},
campaign_id: {$first: "$global_campaign_id"},
device_id: {$first: "$device_id"}
}
}
Then I have this idexes on the collection:
_id: 1
date_created: 1,
global_campaign_id: 1,
global_campaign_id: 1, date_created: 1,
global_campaign_id: 1, device_id: 1, partner_id: 1, date_created: 1
What could be the problem? I need last document for every group. Could sort stage before group help?
Here is result of explain command
{
"stages" : [
{
"$cursor" : {
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "lurity.impressions",
"indexFilterSet" : false,
"parsedQuery" : {
"global_campaign_id" : {
"$in" : [
3151,
3207,
3208,
3209,
3210
]
}
},
"queryHash" : "901B446C",
"planCacheKey" : "06CC82FF",
"winningPlan" : {
"stage" : "PROJECTION_COVERED",
"transformBy" : {
"date_created" : 1,
"device_id" : 1,
"global_campaign_id" : 1,
"partner_id" : 1,
"_id" : 0
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"global_campaign_id" : 1,
"device_id" : 1,
"partner_id" : 1,
"date_created" : 1
},
"indexName" : "global_campaign_id_1_device_id_1_partner_id_1_date_created_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"global_campaign_id" : [ ],
"device_id" : [ ],
"partner_id" : [ ],
"date_created" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"global_campaign_id" : [
"[3151.0, 3151.0]",
"[3207.0, 3207.0]",
"[3208.0, 3208.0]",
"[3209.0, 3209.0]",
"[3210.0, 3210.0]"
],
"device_id" : [
"[MinKey, MaxKey]"
],
"partner_id" : [
"[MinKey, MaxKey]"
],
"date_created" : [
"[MinKey, MaxKey]"
]
}
}
},
"rejectedPlans" : [
{
"stage" : "PROJECTION_SIMPLE",
"transformBy" : {
"date_created" : 1,
"device_id" : 1,
"global_campaign_id" : 1,
"partner_id" : 1,
"_id" : 0
},
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"global_campaign_id" : 1
},
"indexName" : "global_campaign_id_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"global_campaign_id" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"global_campaign_id" : [
"[3151.0, 3151.0]",
"[3207.0, 3207.0]",
"[3208.0, 3208.0]",
"[3209.0, 3209.0]",
"[3210.0, 3210.0]"
]
}
}
}
},
{
"stage" : "PROJECTION_SIMPLE",
"transformBy" : {
"date_created" : 1,
"device_id" : 1,
"global_campaign_id" : 1,
"partner_id" : 1,
"_id" : 0
},
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"global_campaign_id" : 1,
"date_created" : 1
},
"indexName" : "global_campaign_id_1_date_created_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"global_campaign_id" : [ ],
"date_created" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"global_campaign_id" : [
"[3151.0, 3151.0]",
"[3207.0, 3207.0]",
"[3208.0, 3208.0]",
"[3209.0, 3209.0]",
"[3210.0, 3210.0]"
],
"date_created" : [
"[MinKey, MaxKey]"
]
}
}
}
}
]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 304244,
"executionTimeMillis" : 1363,
"totalKeysExamined" : 304246,
"totalDocsExamined" : 0,
"executionStages" : {
"stage" : "PROJECTION_COVERED",
"nReturned" : 304244,
"executionTimeMillisEstimate" : 112,
"works" : 304246,
"advanced" : 304244,
"needTime" : 1,
"needYield" : 0,
"saveState" : 322,
"restoreState" : 322,
"isEOF" : 1,
"transformBy" : {
"date_created" : 1,
"device_id" : 1,
"global_campaign_id" : 1,
"partner_id" : 1,
"_id" : 0
},
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 304244,
"executionTimeMillisEstimate" : 73,
"works" : 304246,
"advanced" : 304244,
"needTime" : 1,
"needYield" : 0,
"saveState" : 322,
"restoreState" : 322,
"isEOF" : 1,
"keyPattern" : {
"global_campaign_id" : 1,
"device_id" : 1,
"partner_id" : 1,
"date_created" : 1
},
"indexName" : "global_campaign_id_1_device_id_1_partner_id_1_date_created_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"global_campaign_id" : [ ],
"device_id" : [ ],
"partner_id" : [ ],
"date_created" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"global_campaign_id" : [
"[3151.0, 3151.0]",
"[3207.0, 3207.0]",
"[3208.0, 3208.0]",
"[3209.0, 3209.0]",
"[3210.0, 3210.0]"
],
"device_id" : [
"[MinKey, MaxKey]"
],
"partner_id" : [
"[MinKey, MaxKey]"
],
"date_created" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 304246,
"seeks" : 2,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
}
},
"nReturned" : NumberLong(304244),
"executionTimeMillisEstimate" : NumberLong(803)
},
{
"$group" : {
"_id" : {
"global_campaign_id" : "$global_campaign_id",
"device_id" : "$device_id",
"partner_id" : "$partner_id"
},
"date_last" : {
"$max" : "$date_created"
},
"partner_id" : {
"$first" : "$partner_id"
},
"campaign_id" : {
"$first" : "$global_campaign_id"
},
"device_id" : {
"$first" : "$device_id"
}
},
"nReturned" : NumberLong(84),
"executionTimeMillisEstimate" : NumberLong(1358)
}
],
"serverInfo" : {
"host" : "nosql-lurity",
"port" : 27017,
"version" : "4.4.13",
"gitVersion" : "df25c71b8674a78e17468f48bcda5285decb9246"
},
"ok" : 1
}
MongoDB not using Index - possible collation issue?
We have a MongoDB Query, fully indexed, that should be scanning at most 4/5 rows. However the query appears to use one only element of the index (the integer) and ignore the string portion.
We are using a case-insensitive collation (strength=2), but it makes no difference if we specify this or not. Documentation: https://docs.mongodb.com/manual/core/index-case-insensitive/
Do collations use indexes? Is there a more efficient way of do we manually need to de-normalise? NOTE: we're not actually using any non-standard characters, the collation is specified purely for case insensitivity.
Version (supports collation):
MongoDB server version: 3.6.13
db.version() => 3.6.13
db.adminCommand( { getParameter: 1, featureCompatibilityVersion: 1 } )
gives:
{
"featureCompatibilityVersion" : {
"version" : "3.6"
},
"ok" : 1,
"operationTime" : Timestamp(1565754388, 51),
"$clusterTime" : {
"clusterTime" : Timestamp(1565754388, 51),
"signature" : {
"hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
"keyId" : NumberLong(0)
}
}
}
DB Structure (summarised) is
{
"PrimaryID": (int)XXX,
"aTables": {
"userExt": [
{
"userExtPlatform": (int)XXX,
"userExtID": (string)XXX,
"userExtActive": (int 1 | 0)XXX,
},
{
"userExtPlatform": (int)XXX,
"userExtID": (string)XXX,
"userExtActive": (int 1 | 0)XXX,
},
...
],
"userOtherData": [
{
"otherDataField1": XXX,
"otherDataField2": XXX,
},
...
],
...
}
}
Index is set up as follows (note - collation is specified as {locale:en, strength: 2}:
{
"v" : 2,
"key" : {
"aTables.userExt.userExtPlatform" : 1,
"aTables.userExt.userExtID" : 1
},
"name" : "extPlatform",
"background" : false,
"ns" : "archive.users",
"collation" : {
"locale" : "en",
"caseLevel" : false,
"caseFirst" : "off",
"strength" : 2,
"numericOrdering" : false,
"alternate" : "non-ignorable",
"maxVariable" : "punct",
"normalization" : false,
"backwards" : false,
"version" : "57.1"
}
}
The query (trying with/without specifying collation)
use archive;
db.users.find(
{
"aTables.userExt.userExtPlatform": 4,
"aTables.userExt.userExtID": "AStringValue",
"aTables.userExt.userExtActive": 1,
"deleted": { "$exists": false }
}
)
db.users.find(
{
"aTables.userExt.userExtPlatform": 4,
"aTables.userExt.userExtID": "AStringValue",
"aTables.userExt.userExtActive": 1,
"deleted": { "$exists": false }
}
).collation( { locale: "en", strength: 2 } )
Note: Removing the 'deleted' clause makes no difference to the speed / results / explain.
Here is the explain, and it shows a vast number of keys and documents queried.
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "archive.users",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"aTables.userExt.userExtActive" : {
"$eq" : 1
}
},
{
"aTables.userExt.userExtID" : {
"$eq" : "PrivateStringRemoved"
}
},
{
"aTables.userExt.userExtPlatform" : {
"$eq" : 4
}
},
{
"$nor" : [
{
"deleted" : {
"$exists" : true
}
}
]
}
]
},
"collation" : {
"locale" : "en",
"caseLevel" : false,
"caseFirst" : "off",
"strength" : 2,
"numericOrdering" : false,
"alternate" : "non-ignorable",
"maxVariable" : "punct",
"normalization" : false,
"backwards" : false,
"version" : "57.1"
},
"winningPlan" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"aTables.userExt.userExtActive" : {
"$eq" : 1
}
},
{
"aTables.userExt.userExtID" : {
"$eq" : "PrivateStringRemoved"
}
},
{
"$nor" : [
{
"deleted" : {
"$exists" : true
}
}
]
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"aTables.userExt.userExtPlatform" : 1,
"aTables.userExt.userExtID" : 1
},
"indexName" : "extPlatform",
"collation" : {
"locale" : "en",
"caseLevel" : false,
"caseFirst" : "off",
"strength" : 2,
"numericOrdering" : false,
"alternate" : "non-ignorable",
"maxVariable" : "punct",
"normalization" : false,
"backwards" : false,
"version" : "57.1"
},
"isMultiKey" : true,
"multiKeyPaths" : {
"aTables.userExt.userExtPlatform" : [
"aTables.userExt"
],
"aTables.userExt.userExtID" : [
"aTables.userExt"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"aTables.userExt.userExtPlatform" : [
"[4.0, 4.0]"
],
"aTables.userExt.userExtID" : [
"[MinKey, MaxKey]"
]
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 0,
"executionTimeMillis" : 1304,
"totalKeysExamined" : 290114,
"totalDocsExamined" : 290114,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"aTables.userExt.userExtActive" : {
"$eq" : 1
}
},
{
"aTables.userExt.userExtID" : {
"$eq" : "PrivateStringRemoved"
}
},
{
"$nor" : [
{
"deleted" : {
"$exists" : true
}
}
]
}
]
},
"nReturned" : 0,
"executionTimeMillisEstimate" : 1245,
"works" : 290115,
"advanced" : 0,
"needTime" : 290114,
"needYield" : 0,
"saveState" : 2267,
"restoreState" : 2267,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 290114,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 290114,
"executionTimeMillisEstimate" : 270,
"works" : 290115,
"advanced" : 290114,
"needTime" : 0,
"needYield" : 0,
"saveState" : 2267,
"restoreState" : 2267,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"aTables.userExt.userExtPlatform" : 1,
"aTables.userExt.userExtID" : 1
},
"indexName" : "extPlatform",
"collation" : {
"locale" : "en",
"caseLevel" : false,
"caseFirst" : "off",
"strength" : 2,
"numericOrdering" : false,
"alternate" : "non-ignorable",
"maxVariable" : "punct",
"normalization" : false,
"backwards" : false,
"version" : "57.1"
},
"isMultiKey" : true,
"multiKeyPaths" : {
"aTables.userExt.userExtPlatform" : [
"aTables.userExt"
],
"aTables.userExt.userExtID" : [
"aTables.userExt"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"aTables.userExt.userExtPlatform" : [
"[4.0, 4.0]"
],
"aTables.userExt.userExtID" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 290114,
"seeks" : 1,
"dupsTested" : 290114,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
"serverInfo" : {
"host" : "api-mdb-archive-03",
"port" : 27017,
"version" : "3.6.13",
"gitVersion" : "db3c76679b7a3d9b443a0e1b3e45ed02b88c539f"
},
"ok" : 1,
"operationTime" : Timestamp(1565753056, 9),
"$clusterTime" : {
"clusterTime" : Timestamp(1565753056, 9),
"signature" : {
"hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
"keyId" : NumberLong(0)
}
}
}
The log output also confirms that it's checking a huge number of documents, and that collation comes through.
2019-08-14T03:23:53.912+0000 I COMMAND [conn20679] command archive.users appName: "MongoDB Shell" command: find { find: "users", filter: { aTables.userExt.userExtPlatform: 4.0, aTables.userExt.userExtID: "PrivateStringRemoved", aTables.userExt.userExtActive: 1.0, deleted: { $exists: false } }, collation: { locale: "en", strength: 2.0 }, lsid: { id: UUID("3178aa31-5ee9-4a79-9848-f01c1842f542") }, $clusterTime: { clusterTime: Timestamp(1565753015, 41), signature: { hash: BinData(0, 0000000000000000000000000000000000000000), keyId: 0 } }, $db: "archive" } planSummary: IXSCAN { aTables.userExt.userExtPlatform: 1, aTables.userExt.userExtID: 1 } keysExamined:289966 docsExamined:289966 cursorExhausted:1 numYields:2267 nreturned:0 reslen:228 locks:{ Global: { acquireCount: { r: 4536 } }, Database: { acquireCount: { r: 2268 } }, Collection: { acquireCount: { r: 2268 } } } protocol:op_msg 1546ms
2019-08-14T03:24:16.864+0000 I COMMAND [conn20679] command archive.users appName: "MongoDB Shell" command: explain { explain: { find: "users", filter: { aTables.userExt.userExtPlatform: 4.0, aTables.userExt.userExtID: "PrivateStringRemoved", aTables.userExt.userExtActive: 1.0, deleted: { $exists: false } }, collation: { locale: "en", strength: 2.0 } }, verbosity: "executionStats", lsid: { id: UUID("3178aa31-5ee9-4a79-9848-f01c1842f542") }, $clusterTime: { clusterTime: Timestamp(1565753033, 128), signature: { hash: BinData(0, 0000000000000000000000000000000000000000), keyId: 0 } }, $db: "archive" } numYields:2267 reslen:3578 locks:{ Global: { acquireCount: { r: 4536 } }, Database: { acquireCount: { r: 2268 } }, Collection: { acquireCount: { r: 2268 } } } protocol:op_msg 1341ms
For completeness, this is driven by PHP, but verified in Mongo CMD as above. Here is the PHP:
$aParams = [
'aTables.userExt.userExtID' => 4,
'aTables.userExt.userExtPlatform' => 'PrivateStringRemoved',
'aTables.userExt.userExtActive' => 1,
'deleted': [
'$exists' => false
]
];
$aOptions = [
'readPreference' => new \MongoDB\Driver\ReadPreference(\MongoDB\Driver\ReadPreference::RP_NEAREST),
'skip' => $start,
'limit' => $limit,
'typeMap' => [
'root' => 'array',
'document' => 'array',
'array' => 'array'
],
'collation' => [
'locale' => 'en',
'strength' => 2
],
];
try {
$aResults = $collectionArchive->find($aParams, $aOptions);
} catch (\Exception $exception) {
throw new ArchiverException('Mongo Error', ArchiverRequest::ERROR_MONGO, $exception->getMessage());
}
Posting here, for reassurance if anyone searches. (Based on other answers)
Having played around, the following syntax is the correct one. You need to group the final elements into $elemMatch as below.
db.users.find(
{
"aTables.userExt" : {
"$elemMatch" : {
"userExtPlatform": 4,
"userExtID": "AStringValue",
"userExtActive": 1
}
}
}
).collation( { locale: "en", strength: 2 } ).explain("executionStats")
As requested: here is the explain:
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "archive.users",
"indexFilterSet" : false,
"parsedQuery" : {
"aTables.userExt" : {
"$elemMatch" : {
"$and" : [
{
"userExtActive" : {
"$eq" : 1
}
},
{
"userExtID" : {
"$eq" : "AStringValue"
}
},
{
"userExtPlatform" : {
"$eq" : 4
}
}
]
}
}
},
"collation" : {
"locale" : "en",
"caseLevel" : false,
"caseFirst" : "off",
"strength" : 2,
"numericOrdering" : false,
"alternate" : "non-ignorable",
"maxVariable" : "punct",
"normalization" : false,
"backwards" : false,
"version" : "57.1"
},
"winningPlan" : {
"stage" : "FETCH",
"filter" : {
"aTables.userExt" : {
"$elemMatch" : {
"$and" : [
{
"userExtPlatform" : {
"$eq" : 4
}
},
{
"userExtID" : {
"$eq" : "AStringValue"
}
},
{
"userExtActive" : {
"$eq" : 1
}
}
]
}
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"aTables.userExt.userExtPlatform" : 1,
"aTables.userExt.userExtID" : 1
},
"indexName" : "extPlatform",
"collation" : {
"locale" : "en",
"caseLevel" : false,
"caseFirst" : "off",
"strength" : 2,
"numericOrdering" : false,
"alternate" : "non-ignorable",
"maxVariable" : "punct",
"normalization" : false,
"backwards" : false,
"version" : "57.1"
},
"isMultiKey" : true,
"multiKeyPaths" : {
"aTables.userExt.userExtPlatform" : [
"aTables.userExt"
],
"aTables.userExt.userExtID" : [
"aTables.userExt"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"aTables.userExt.userExtPlatform" : [
"[4.0, 4.0]"
],
"aTables.userExt.userExtID" : [
"[\")MOK9C5S)?Q1\u0001\u0010\", \")MOK9C5S)?Q1\u0001\u0010\"]"
]
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 0,
"executionTimeMillis" : 4,
"totalKeysExamined" : 0,
"totalDocsExamined" : 0,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"aTables.userExt" : {
"$elemMatch" : {
"$and" : [
{
"userExtPlatform" : {
"$eq" : 4
}
},
{
"userExtID" : {
"$eq" : "AStringValue"
}
},
{
"userExtActive" : {
"$eq" : 1
}
}
]
}
}
},
"nReturned" : 0,
"executionTimeMillisEstimate" : 0,
"works" : 1,
"advanced" : 0,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 0,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 0,
"executionTimeMillisEstimate" : 0,
"works" : 1,
"advanced" : 0,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"aTables.userExt.userExtPlatform" : 1,
"aTables.userExt.userExtID" : 1
},
"indexName" : "extPlatform",
"collation" : {
"locale" : "en",
"caseLevel" : false,
"caseFirst" : "off",
"strength" : 2,
"numericOrdering" : false,
"alternate" : "non-ignorable",
"maxVariable" : "punct",
"normalization" : false,
"backwards" : false,
"version" : "57.1"
},
"isMultiKey" : true,
"multiKeyPaths" : {
"aTables.userExt.userExtPlatform" : [
"aTables.userExt"
],
"aTables.userExt.userExtID" : [
"aTables.userExt"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"aTables.userExt.userExtPlatform" : [
"[4.0, 4.0]"
],
"aTables.userExt.userExtID" : [
"[\")MOK9C5S)?Q1\u0001\u0010\", \")MOK9C5S)?Q1\u0001\u0010\"]"
]
},
"keysExamined" : 0,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
"serverInfo" : {
"host" : "api-mdb-archive-03",
"port" : 27017,
"version" : "3.6.13",
"gitVersion" : "db3c76679b7a3d9b443a0e1b3e45ed02b88c539f"
},
"ok" : 1,
"operationTime" : Timestamp(1565870195, 8),
"$clusterTime" : {
"clusterTime" : Timestamp(1565870195, 8),
"signature" : {
"hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
"keyId" : NumberLong(0)
}
}
}
My collections are like this:
{
"_index" : "test_index",
"_type" : "test_type",
"_id" : "10000",
"_score" : 1.0,
"_source" : {
"user_id" : 12,
"index_date" : {
"date" : "2018-02-06 14:25:49.816952",
"timezone_type" : 3,
"timezone" : "UTC"
},
"rating" : null,
"orders" : [
{
"hour" : "08",
"count" : 1
},
{
"hour" : "10",
"count" : 1
}
],
"products" : [
{
"p_id" : 970111,
"count" : 4
},
{
"p_id" : 1280811,
"count" : 1
},
]
}
},
and tried to access to {"hour":"10"}
My query is:
$query = new Query\Nested();
$query->setPath('orders');
$term = new Term();
$term->setTerm('orders.hour', $order->getCreatedAt()->format('H'));
$query->setQuery($term);
dump($finder->find($query));die;
but i got the following error:
[Elastica\Exception\ResponseException]
failed to create query: {
"nested" : {
"query" : {
"term" : {
"orders.hour" : {
"value" : "12",
"boost" : 1.0
}
}
},
"path" : "orders",
"ignore_unmapped" : false,
"score_mode" : "avg",
"boost" : 1.0
}
} [index: test_index] [reason: all shards failed]
Your documents not look like nested queries.
I assume that finder is your repository manager that is defined as orders repository, your code should look something like this
$finder = $this->get('fos_elastica.repository_manager')->getRepository('YourBundle:order');
$boolquery = new Query\BoolQuery();
$term = new Query\Term();
$term->setTerm('hour', $order->getCreatedAt()->format('H'));
$boolquery->addMust($term);
$finder->find($boolquery);
I have MongoDB Collection with following data
MongoDB Enterprise > db.UCODEPatter.find()
{ "_id" : { "UCODE" : NumberLong("233220524641368") }, "value" : 117 }
{ "_id" : { "UCODE" : NumberLong("233222325602424") }, "value" : 71 }
{ "_id" : { "UCODE" : NumberLong("233222325601932") }, "value" : 69 }
{ "_id" : { "UCODE" : NumberLong("233222325601947") }, "value" : 59 }
{ "_id" : { "UCODE" : NumberLong("233222325602414") }, "value" : 58 }
{ "_id" : { "UCODE" : NumberLong("233222325602419") }, "value" : 49 }
{ "_id" : { "UCODE" : NumberLong("233222325602418") }, "value" : 48 }
{ "_id" : { "UCODE" : NumberLong("233220535710165") }, "value" : 47 }
{ "_id" : { "UCODE" : NumberLong("233222325602405") }, "value" : 46 }
{ "_id" : { "UCODE" : NumberLong("233222325602420") }, "value" : 45 }
{ "_id" : { "UCODE" : NumberLong("233220537026120") }, "value" : 45 }
{ "_id" : { "UCODE" : NumberLong("233222686003297") }, "value" : 44 }
{ "_id" : { "UCODE" : NumberLong("233222325602417") }, "value" : 44 }
i use below php code for data serching
<?php
// Configuration
$dbhost = 'localhost';
$dbname = 'EIRLOGData';
$m = new Mongo("mongodb://$dbhost");
$db = $m->$dbname;
$c_users = $db->UCODEPatter;
$user = array(
"_id" => "\"UCODE\" : NumberLong(\"233220524641368\")"
);
$user = $c_users->findOne($user);
var_dump($user);
?>
when i run it , it return no data , please help me to sort this
I am not too sure why you are storing { "UCODE" : NumberLong("233220524641368") }, "value" : 117 } in your _id, but you could try the following:
$user = array(
"_id.UCODE" => 233220524641368
);
I'm trying to create a mongo query which will return results where all arrays have a specific element set to false.
An example data record :-
images: [
{
id: ObjectId("516bef7fc05e877b31000000"),
primary: true
},
{
id: ObjectId("516bef2ac05e879622000000"),
primary: false
},
{
id: ObjectId("516beeb7c05e879e2a000000"),
primary: false
}
],
name: "test",
etc: "etc"
I only wish to find documents where all primary fields are set to false however normally (using no query selectors or elemMatch) mongo will return this document because at least 1 of the array elements match.
How would I make mongo only return documents where they all match my search parameters?
Many thanks.
You can do this with the aggregation framework quite easily:
db.so.aggregate( [
{ $unwind: "$images" },
{ $group: {
_id: '$_id',
all: { $sum: 1 },
all_primary: { $sum: { $cond: [ { $eq: [ '$images.primary', true ] }, 1, 0 ] } },
images: { $push: '$images' },
name: { $first: '$name' },
etc: { $first: '$etc' },
} },
{ $project: {
_id: 1,
images: 1,
name: 1,
etc: 1,
same: { $cond: [ { $eq: [ '$all', '$all_primary' ] }, 1, 0 ] }
} },
{ $match: { 'same' : 1 } }
] );
With this as input:
{
"_id" : ObjectId("5203730bf8eaa52a846ebc3e"),
"images" : [
{
"id" : ObjectId("516bef7fc05e877b31000000"),
"primary" : true
},
{
"id" : ObjectId("516bef2ac05e879622010000"),
"primary" : true
},
{
"id" : ObjectId("516beeb7c05e879e2a000010"),
"primary" : true
}
],
"name" : "Derick",
"Etc" : true
}
{
"_id" : ObjectId("52037315f8eaa52a846ebc3f"),
"images" : [
{
"id" : ObjectId("516bef7fc05e877b31000000"),
"primary" : true
},
{
"id" : ObjectId("516bef2ac05e879622010000"),
"primary" : true
},
{
"id" : ObjectId("516beeb7c05e879e2a000020"),
"primary" : false
}
],
"name" : "James",
"Etc" : true
}
{
"_id" : ObjectId("520373621a78238235b6ffbf"),
"images" : [
{
"id" : ObjectId("516bef7fc05e877b31000000"),
"primary" : true
},
{
"id" : ObjectId("516bef2ac05e879622010000"),
"primary" : true
},
{
"id" : ObjectId("516beeb7c05e879e2a000020"),
"primary" : false
}
],
"name" : "James",
"etc" : true
}
{
"_id" : ObjectId("5203736b1a78238235b6ffc0"),
"images" : [
{
"id" : ObjectId("516bef7fc05e877b31000000"),
"primary" : true
},
{
"id" : ObjectId("516bef2ac05e879622010000"),
"primary" : true
},
{
"id" : ObjectId("516beeb7c05e879e2a000020"),
"primary" : true
}
],
"name" : "James",
"etc" : true
}
This outputs:
{
"result" : [
{
"_id" : ObjectId("5203736b1a78238235b6ffc0"),
"images" : [
{
"id" : ObjectId("516bef7fc05e877b31000000"),
"primary" : true
},
{
"id" : ObjectId("516bef2ac05e879622010000"),
"primary" : true
},
{
"id" : ObjectId("516beeb7c05e879e2a000020"),
"primary" : true
}
],
"name" : "James",
"etc" : true,
"same" : 1
},
{
"_id" : ObjectId("5203730bf8eaa52a846ebc3e"),
"images" : [
{
"id" : ObjectId("516bef7fc05e877b31000000"),
"primary" : true
},
{
"id" : ObjectId("516bef2ac05e879622010000"),
"primary" : true
},
{
"id" : ObjectId("516beeb7c05e879e2a000010"),
"primary" : true
}
],
"name" : "Derick",
"etc" : null,
"same" : 1
}
],
"ok" : 1
}
Wouldn't it be much simpler to exclude all documents where images has primary:true element?
{ "images" :
{ "$not" :
{"$elemMatch" : { "primary" : true }}
}
}
Naturally, this is only applicable to a boolean nested field, as in this case.
Having sort trouble with a similar kind of query...But just look at Dericks code, the docs say not to use $first within a $group operation, unless the $group operation is immediately after a $sort.
http://docs.mongodb.org/manual/reference/operator/aggregation/first/#grp._S_first
Presuming Images is a document you could use(from the shell):
db.images.find({'primary':'false'})
If Images is an object:
db.mydoc.find({'images':{'primary':'false'}})