Re: Query optimisation: count() with quite slow

From: Asya Kamsky <asya@xxxxxxxxxxx>
To: mongodb-user <mongodb-user@xxxxxxxxxxxxxxxx>
Date: Wed, 11 May 2016 11:34:48 -0400
Why ads?
Your query syntax is likely not what you want.  If
scheduleInfo.stopSchedules is an array, then you want to use $elemMatch to
indicate that you want the same array element to match both $gte and $lte
conditions.

See more here:
https://docs.mongodb.com/manual/reference/operator/query/elemMatch/

Asya




On Wed, May 11, 2016 at 3:24 AM, 'azzy _home' via mongodb-user <
mongodb-user@xxxxxxxxxxxxxxxx> wrote:

(Repost of
https://groups.google.com/d/msg/mongodb-csharp/CTCuFi7QjMA/jwWGMGG1JAAJ)

I have been scratching my head over this problem long now. My Count query
takes aroud ~500ms for 5000 documents and I hope someone can help me
optimise it.

Following is my query:

var from =  NumberLong(Date.parse('2016-01-01T00:00:00.0000000+02:00')/
1000);
var until = NumberLong(Date.parse('2016-04-30T00:00:00.0000000+02:00')/
1000);



db.tourCloudTest2.count({ $and: [{
'scheduleInfo.stopSchedules.lastETAResultUnix':{$gte: from, $lte: until
}},{'tour.stops.locationId': {$exists: 1}}]});




My collection contains following schema:


{
  "tour": {
    "stops": [ // stop with locationId
      "locationId": "bla.."
    ],[
      // stop without locationId
    ]
  }"scheduleInfo": {
    "stopSchedules": [
      "lastETAResultUnix": NumberLong("date...")
    ]
  }
}



The above query takes me around 500ms for ~5000 documents without using any index on "lastETAResultUnix" field. Expain(true) with the query gives me this...



{
  "queryPlanner": {
    "plannerVersion": 1,
    "namespace": "TestUnixDate.tourCloudTest2",
    "indexFilterSet": false,
    "parsedQuery": {
      "$and": [
        {
          "scheduleInfo.stopSchedules.lastETAResultUnix": {
            "$lte": 1461967200
          }
        },
        {
          "scheduleInfo.stopSchedules.lastETAResultUnix": {
            "$gte": -62135596800
          }
        },
        {
          "tour.stops.locationId": {
            "$exists": true
          }
        }
      ]
    },
    "winningPlan": {
      "stage": "COLLSCAN",
      "filter": {
        "$and": [
          {
            "scheduleInfo.stopSchedules.lastETAResultUnix": {
              "$lte": 1461967200
            }
          },
          {
            "scheduleInfo.stopSchedules.lastETAResultUnix": {
              "$gte": -62135596800
            }
          },
          {
            "tour.stops.locationId": {
              "$exists": true
            }
          }
        ]
      },
      "direction": "forward"
    },
    "rejectedPlans": [

    ]
  },
  "executionStats": {
    "executionSuccess": true,
    "nReturned": 5661,
    "executionTimeMillis": 827,
    "totalKeysExamined": 0,
    "totalDocsExamined": 45302,
    "executionStages": {
      "stage": "COLLSCAN",
      "filter": {
        "$and": [
          {
            "scheduleInfo.stopSchedules.lastETAResultUnix": {
              "$lte": 1461967200
            }
          },
          {
            "scheduleInfo.stopSchedules.lastETAResultUnix": {
              "$gte": -62135596800
            }
          },
          {
            "tour.stops.locationId": {
              "$exists": true
            }
          }
        ]
      },
      "nReturned": 5661,
      "executionTimeMillisEstimate": 780,
      "works": 45304,
      "advanced": 5661,
      "needTime": 39642,
      "needFetch": 0,
      "saveState": 353,
      "restoreState": 353,
      "isEOF": 1,
      "invalidates": 0,
      "direction": "forward",
      "docsExamined": 45302
    },
    "allPlansExecution": [

    ]
  },
  "serverInfo": {
    "host": "DAve-build-1",
    "port": 27017,
    "version": "3.0.6",
    "gitVersion": "nogitversion"
  },
  "ok": 1
}

Performance using an index on the lastETAResult field is horrible. Again with explain ...


{
  "queryPlanner": {
    "plannerVersion": 1,
    "namespace": "TestUnixDate.tourCloudTest2",
    "indexFilterSet": false,
    "parsedQuery": {
      "$and": [
        {
          "scheduleInfo.stopSchedules.lastETAResultUnix": {
            "$lte": 1461967200
          }
        },
        {
          "scheduleInfo.stopSchedules.lastETAResultUnix": {
            "$gte": -62135596800
          }
        },
        {
          "tour.stops.locationId": {
            "$exists": true
          }
        }
      ]
    },
    "winningPlan": {
      "stage": "KEEP_MUTATIONS",
      "inputStage": {
        "stage": "FETCH",
        "filter": {
          "$and": [
            {
              "scheduleInfo.stopSchedules.lastETAResultUnix": {
                "$gte": -62135596800
              }
            },
            {
              "tour.stops.locationId": {
                "$exists": true
              }
            }
          ]
        },
        "inputStage": {
          "stage": "IXSCAN",
          "keyPattern": {
            "scheduleInfo.stopSchedules.lastETAResultUnix": 1
          },
          "indexName": "scheduleInfo.stopSchedules.lastETAResultUnix_1",
          "isMultiKey": true,
          "direction": "forward",
          "indexBounds": {
            "scheduleInfo.stopSchedules.lastETAResultUnix": [
              "[-1.#INF, 1461967200.0]"
            ]
          }
        }
      }
    },
    "rejectedPlans": [

    ]
  },
  "executionStats": {
    "executionSuccess": true,
    "nReturned": 5661,
    "executionTimeMillis": 4631,
    "totalKeysExamined": 76868,
    "totalDocsExamined": 45065,
    "executionStages": {
      "stage": "KEEP_MUTATIONS",
      "nReturned": 5661,
      "executionTimeMillisEstimate": 4380,
      "works": 76869,
      "advanced": 5661,
      "needTime": 71207,
      "needFetch": 0,
      "saveState": 609,
      "restoreState": 609,
      "isEOF": 1,
      "invalidates": 0,
      "inputStage": {
        "stage": "FETCH",
        "filter": {
          "$and": [
            {
              "scheduleInfo.stopSchedules.lastETAResultUnix": {
                "$gte": -62135596800
              }
            },
            {
              "tour.stops.locationId": {
                "$exists": true
              }
            }
          ]
        },
        "nReturned": 5661,
        "executionTimeMillisEstimate": 4360,
        "works": 76869,
        "advanced": 5661,
        "needTime": 71207,
        "needFetch": 0,
        "saveState": 609,
        "restoreState": 609,
        "isEOF": 1,
        "invalidates": 0,
        "docsExamined": 45065,
        "alreadyHasObj": 0,
        "inputStage": {
          "stage": "IXSCAN",
          "nReturned": 45065,
          "executionTimeMillisEstimate": 3750,
          "works": 76869,
          "advanced": 45065,
          "needTime": 31803,
          "needFetch": 0,
          "saveState": 609,
          "restoreState": 609,
          "isEOF": 1,
          "invalidates": 0,
          "keyPattern": {
            "scheduleInfo.stopSchedules.lastETAResultUnix": 1
          },
          "indexName": "scheduleInfo.stopSchedules.lastETAResultUnix_1",
          "isMultiKey": true,
          "direction": "forward",
          "indexBounds": {
            "scheduleInfo.stopSchedules.lastETAResultUnix": [
              "[-1.#INF, 1461967200.0]"
            ]
          },
          "keysExamined": 76868,
          "dupsTested": 76868,
          "dupsDropped": 31803,
          "seenInvalidated": 0,
          "matchTested": 0
        }
      }
    },
    "allPlansExecution": [

    ]
  },
  "serverInfo": {
    "host": "DAve-build-1",
    "port": 27017,
    "version": "3.0.6",
    "gitVersion": "nogitversion"
  },
  "ok": 1
}


Thank you for any help.

--
You received this message because you are subscribed to the Google Groups
"mongodb-user"
group.

For other MongoDB technical support options, see:
https://docs.mongodb.org/manual/support/
---
You received this message because you are subscribed to the Google Groups
"mongodb-user" group.
To unsubscribe from this group and stop receiving emails from it, send an
email to mongodb-user+unsubscribe@xxxxxxxxxxxxxxxx.
To post to this group, send email to mongodb-user@xxxxxxxxxxxxxxxx.
Visit this group at https://groups.google.com/group/mongodb-user.
To view this discussion on the web visit
https://groups.google.com/d/msgid/mongodb-user/fa6db41f-b282-45fb-a5e7-b98485055c03%40googlegroups.com
<https://groups.google.com/d/msgid/mongodb-user/fa6db41f-b282-45fb-a5e7-b98485055c03%40googlegroups.com?utm_medium=email&utm_source=footer>
.
For more options, visit https://groups.google.com/d/optout.




-- 
Asya Kamsky
Lead Product Manager
MongoDB
Download MongoDB - mongodb.org/downloads
Free MongoDB Monitoring - cloud.mongodb.com
Free Online Education - university.mongodb.com
Get Involved - mongodb.org/community
We're Hiring! - https://www.mongodb.com/careers

-- 
You received this message because you are subscribed to the Google Groups "mongodb-user"
group.

For other MongoDB technical support options, see: https://docs.mongodb.org/manual/support/
--- 
You received this message because you are subscribed to the Google Groups "mongodb-user" group.
To unsubscribe from this group and stop receiving emails from it, send an email to mongodb-user+unsubscribe@xxxxxxxxxxxxxxxx.
To post to this group, send email to mongodb-user@xxxxxxxxxxxxxxxx.
Visit this group at https://groups.google.com/group/mongodb-user.
To view this discussion on the web visit https://groups.google.com/d/msgid/mongodb-user/CAOe6dJBHV%2B1RRemaDXf-iBs9S3s2wLiVnXJJnFnt%2B4UX7i_nTw%40mail.gmail.com.
For more options, visit https://groups.google.com/d/optout.
Why ads?