小贝子编程

通过聚合框架或map/reduce将事件数据嵌入MongoDB中的文档



我有以下数据结构，并希望将包含loc字段的文档嵌入到包含持续时间的文档中，但仅当时间戳在时间戳(ts)减去父文档的以秒为单位的持续时间内时。这是可能的聚合框架或其他与mapreduce ?

{
    "_id" : ObjectId("53df2a44e6583c76253c9869"),
    "deviceId" : NumberLong(1377700226807),
    "ts" : ISODate("2014-08-04T08:37:55.000Z"),
    "duration" : NumberLong(1642),
}
{
    "_id" : ObjectId("53df2a41e6583c4e243c9869"),
    "deviceId" : NumberLong(1377700226807),
    "ts" : ISODate("2014-08-04T08:37:53.000Z"),
    "loc" : {
        "lon" : 5.1101453,
        "lat" : 52.0625047
    }
}
{
    "_id" : ObjectId("53df2a3fe6583c38203c986a"),
    "deviceId" : NumberLong(1377700226807),
    "ts" : ISODate("2014-08-04T08:37:50.000Z"),
        "loc" : {
            "lon" : 5.1101297,
            "lat" : 52.0625031
        }
}
{
    "_id" : ObjectId("53df2a44e6583c76253c9869"),
    "deviceId" : NumberLong(1377700226807),
    "ts" : ISODate("2014-08-04T06:37:55.000Z"),
    "duration" : NumberLong(3600),
}
{
    "_id" : ObjectId("53df2a38e6583c03253c9869"),
    "deviceId" : NumberLong(1377700226807),
    "ts" : ISODate("2014-08-04T06:37:44.000Z"),
        "loc" : {
            "lon" : 5.1101176,
            "lat" : 52.0625171
        }
}
{
    "_id" : ObjectId("53df2a33e6583c51243c9869"),
    "deviceId" : NumberLong(1377700226807),
    "ts" : ISODate("2014-08-04T06:37:38.000Z"),
        "loc" : {
            "lon" : 5.1101409,
            "lat" : 52.0625818
        }
}
{
    "_id" : ObjectId("53df2a2de6583c38203c9869"),
    "deviceId" : NumberLong(1377700226807),
    "ts" : ISODate("2014-08-04T06:37:32.000Z"),
        "loc" : {
            "lon" : 5.1099513,
            "lat" : 52.0624157
        }
}

这是所需的格式

{
    "_id" : ObjectId("53df2a44e6583c76253c9869"),
    "deviceId" : NumberLong(1377700226807),
    "ts" : ISODate("2014-08-04T08:37:55.000Z"),
    "duration" : NumberLong(1642),
    "data" : [
       {
          "ts" : ISODate("2014-08-04T08:37:53.000Z"),
          "loc" : {
             "lon" : 5.1101453,
             "lat" : 52.0625047
          }
       },
       {
          "ts" : ISODate("2014-08-04T08:37:50.000Z"),
          "loc" : {
            "lon" : 5.1101297,
            "lat" : 52.0625031
           }
       }
    ]
}

这在Aggregation框架中是不容易做到的，但是在MapReduce中可以做到。

假设数据是属性收集的(即，对于具有"loc"值的文档，没有缺少带有持续时间的"ride"文档)，您可以这样做:

map=function () {
   var startTime;
   if (this.hasOwnProperty("duration"))
       startTime=this.ts-this.duration*1000;
   else
       startTime=this.ts;
   emit(this.deviceId, {startTs:new Date(startTime), endTs:this.ts, loc:this.loc, duration:this.duration});
}

Map以规格化格式输出，每个deviceId减少到一个数组。

reduce=function (key,values) {
   var result = { vals : [ ] };
   values.forEach(function(v) {
       result.vals.push(v);
   })
   return result;
}

所有实际的处理(对每个deviceId进行分组)都发生在finalize函数中，该函数为每个deviceId获取一个数组，并对其进行排序，并将其分组到您期望的文档中。

finalize=function (key, value) {
    var lastI=-1;
    var result = {rides: [ ] };
    var ride = { };
    value.vals.sort(function(a,b) { return a.startTs.getTime() - b.startTs.getTime(); } );
    for (i=0; i<value.vals.length; i++) {
        if (value.vals[i].loc == null ) {
           if (ride.hasOwnProperty("locations")) {
               result.rides.push(ride);
               ride={};
           }
           ride["start"]=value.vals[i].startTs;
           ride["end"]=value.vals[i].endTs;
           ride["duration"]=value.vals[i].duration;
           ride["locations"]=[];
           lastI=i;
        } else {
           ride.locations.push({ loc: value.vals[i].loc, ts: value.vals[i].endTs});
        }
    }
    result.rides.push(ride);
    return result;
}

我添加了一对deviceid到您的测试数据:

db.rides.find({},{_id:0})
{ "deviceId" : NumberLong("1377700226807"), "ts" : ISODate("2014-08-04T06:37:32Z"), "loc" : { "lon" : 5.1099513, "lat" : 52.0624157 } }
{ "deviceId" : NumberLong("1377700226910"), "ts" : ISODate("2014-08-04T06:37:32Z"), "loc" : { "lon" : 5.1099513, "lat" : 52.0624157 } }
{ "deviceId" : NumberLong("1377700226807"), "ts" : ISODate("2014-08-04T06:37:38Z"), "loc" : { "lon" : 5.1101409, "lat" : 52.0625818 } }
{ "deviceId" : NumberLong("1377700226910"), "ts" : ISODate("2014-08-04T06:37:38Z"), "loc" : { "lon" : 5.1101409, "lat" : 52.0625818 } }
{ "deviceId" : NumberLong("1377700226807"), "ts" : ISODate("2014-08-04T06:37:44Z"), "loc" : { "lon" : 5.1101176, "lat" : 52.0625171 } }
{ "deviceId" : NumberLong("1377700226910"), "ts" : ISODate("2014-08-04T06:37:44Z"), "loc" : { "lon" : 5.1101176, "lat" : 52.0625171 } }
{ "deviceId" : NumberLong("1377700226807"), "ts" : ISODate("2014-08-04T06:37:55Z"), "duration" : NumberLong(3600) }
{ "deviceId" : NumberLong("1377700226910"), "ts" : ISODate("2014-08-04T06:37:55Z"), "duration" : NumberLong(3600) }
{ "deviceId" : NumberLong("1377700226807"), "ts" : ISODate("2014-08-04T08:37:50Z"), "loc" : { "lon" : 5.1101297, "lat" : 52.0625031 } }
{ "deviceId" : NumberLong("1377700226908"), "ts" : ISODate("2014-08-04T08:37:50Z"), "loc" : { "lon" : 5.1101297, "lat" : 52.0625031 } }
{ "deviceId" : NumberLong("1377700226807"), "ts" : ISODate("2014-08-04T08:37:53Z"), "loc" : { "lon" : 5.1101453, "lat" : 52.0625047 } }
{ "deviceId" : NumberLong("1377700226908"), "ts" : ISODate("2014-08-04T08:37:53Z"), "loc" : { "lon" : 5.1101453, "lat" : 52.0625047 } }
{ "deviceId" : NumberLong("1377700226807"), "ts" : ISODate("2014-08-04T08:37:55Z"), "duration" : NumberLong(1642) }
{ "deviceId" : NumberLong("1377700226908"), "ts" : ISODate("2014-08-04T08:37:55Z"), "duration" : NumberLong(1642) }

并在MR

中运行
db.rides.mapReduce(map, reduce, {out:"newrides", finalize:finalize})
{
        "result" : "frides",
        "timeMillis" : 47,
        "counts" : {
                "input" : 14,
                "emit" : 14,
                "reduce" : 3,
                "output" : 3
        },
        "ok" : 1
}

结果:

db.newrides.find().pretty()
{
        "_id" : NumberLong("1377700226807"),
        "value" : {
                "rides" : [
                        {
                                "start" : ISODate("2014-08-04T05:37:55Z"),
                                "end" : ISODate("2014-08-04T06:37:55Z"),
                                "duration" : NumberLong(3600),
                                "locations" : [
                                        {
                                                "loc" : {
                                                        "lon" : 5.1099513,
                                                        "lat" : 52.0624157
                                                },
                                                "ts" : ISODate("2014-08-04T06:37:32Z")
                                        },
                                        {
                                                "loc" : {
                                                        "lon" : 5.1101409,
                                                        "lat" : 52.0625818
                                                },
                                                "ts" : ISODate("2014-08-04T06:37:38Z")
                                        },
                                        {
                                                "loc" : {
                                                        "lon" : 5.1101176,
                                                        "lat" : 52.0625171
                                                },
                                                "ts" : ISODate("2014-08-04T06:37:44Z")
                                        }
                                ]
                        },
                        {
                                "start" : ISODate("2014-08-04T08:10:33Z"),
                                "end" : ISODate("2014-08-04T08:37:55Z"),
                                "duration" : NumberLong(1642),
                                "locations" : [
                                        {
                                                "loc" : {
                                                        "lon" : 5.1101297,
                                                        "lat" : 52.0625031
                                                },
                                                "ts" : ISODate("2014-08-04T08:37:50Z")
                                        },
                                        {
                                                "loc" : {
                                                        "lon" : 5.1101453,
                                                        "lat" : 52.0625047
                                                },
                                                "ts" : ISODate("2014-08-04T08:37:53Z")
                                        }
                                ]
                        }
                ]
        }
}
{
        "_id" : NumberLong("1377700226908"),
        "value" : {
                "rides" : [
                        {
                                "start" : ISODate("2014-08-04T08:10:33Z"),
                                "end" : ISODate("2014-08-04T08:37:55Z"),
                                "duration" : NumberLong(1642),
                                "locations" : [
                                        {
                                                "loc" : {
                                                        "lon" : 5.1101297,
                                                        "lat" : 52.0625031
                                                },
                                                "ts" : ISODate("2014-08-04T08:37:50Z")
                                        },
                                        {
                                                "loc" : {
                                                        "lon" : 5.1101453,
                                                        "lat" : 52.0625047
                                                },
                                                "ts" : ISODate("2014-08-04T08:37:53Z")
                                        }
                                ]
                        }
                ]
        }
}
{
        "_id" : NumberLong("1377700226910"),
        "value" : {
                "rides" : [
                        {
                                "start" : ISODate("2014-08-04T05:37:55Z"),
                                "end" : ISODate("2014-08-04T06:37:55Z"),
                                "duration" : NumberLong(3600),
                                "locations" : [
                                        {
                                                "loc" : {
                                                        "lon" : 5.1099513,
                                                        "lat" : 52.0624157
                                                },
                                                "ts" : ISODate("2014-08-04T06:37:32Z")
                                        },
                                        {
                                                "loc" : {
                                                        "lon" : 5.1101409,
                                                        "lat" : 52.0625818
                                                },
                                                "ts" : ISODate("2014-08-04T06:37:38Z")
                                        },
                                        {
                                                "loc" : {
                                                        "lon" : 5.1101176,
                                                        "lat" : 52.0625171
                                                },
                                                "ts" : ISODate("2014-08-04T06:37:44Z")
                                        }
                                ]
                        }
                ]
        }
}

聚合框架无法在处理管道时跨文档"保留"信息。你所描述的那种"父/子"关系，即该元素是通过与"父"文档的比较来决定的，而"父"文档没有在文档中直接指定，因此在这里是不可能的。

mapReduce方法可以访问"全局"作用域变量。这样就可以"识别"父节点的详细信息，这些信息可以存储在一个变量中，以便根据需要与可能的子节点进行比较。

db.collection.mapReduce(
  function() {
    lastValue.data = [];
    if ( lastId == null || this.hasOwnProperty("duration") ) {
      lastId = this._id;
      lastValue.deviceId = this.deviceId;
      lastValue.ts = this.ts;
      lastValue.duration = this.duration;
    }
    if (
        ( this.hasOwnProperty("loc") ) &&
        (
          ( lastValue.ts.valueOf() - ( lastValue.duration * 1000 ) ) <
          this.ts.valueOf()
        )
      )
      lastValue.data.push({
        "ts": this.ts,
        "loc": this.loc
      });

    emit ( lastId, lastValue );
  },
  function (key,values) {
    var reduced = {};
    values.forEach(function(value) {
      if ( !reduced.hasOwnProperty("deviceId") ) {
        reduced.deviceId = value.deviceId;
        reduced.ts = value.ts;
        reduced.duration = value.duration;
        reduced.data = [];
      }
      value.data.forEach(function(dat) {
        reduced.data.push(dat);
      });
    });
    return reduced;
  },
  { 
      "sort": { "ts": -1 },
      "scope": { "lastId": null, "lastValue": {} }, 
      "out": { "inline": 1 }
  }
)

所以本质上，这是"存储"要发出的"键"为"lastd"，使用mapReduce可用的"范围"声明。清除"父"文档包含持续时间，以便在这里有一些东西可以使用。

"data"元素需要以数组形式发出。这是由于文档中描述的"reduce"函数的需求。关键是输入必须与期望的输入格式相同。每个"键"可以多次调用"reduce"函数。

"mapper"中的条件可以在调用"reduce"函数之前限制该数组中存在的值。这也避免了在只有一个"键"发出的情况下依赖于添加"finalize"方法，并且"reduce"函数不会触及该项。这样就减少了前期过滤的工作量。

"reducer"现在被降级为只是"合并"结果，以组合"data"中有资格作为"子"包含的元素。

排序顺序自然是按时间戳"降序"或"ts"值排序。这将是一个很好的索引点，并且顺序与处理文档进行比较的方式保持一致，因此在检测到新父节点时，每个"断点"都是有效的。

输出当然是mapReduce样式。所以要得到你想要的"精确"输出需要更多的后期处理，但正如你所看到的，它基本上是结果:

    "results" : [
        {
            "_id" : ObjectId("53f15b8beb75bdce84f914a1"),
            "value" : {
                "deviceId" : NumberLong("1377700226807"),
                "ts" : ISODate("2014-08-04T08:37:55Z"),
                "duration" : NumberLong(1642),
                "data" : [
                    {
                        "ts" : ISODate("2014-08-04T08:37:53Z"),
                        "loc" : {
                            "lon" : 5.1101453,
                            "lat" : 52.0625047
                        }
                    },
                    {
                        "ts" : ISODate("2014-08-04T08:37:50Z"),
                        "loc" : {
                            "lon" : 5.1101297,
                            "lat" : 52.0625031
                        }
                    }
                ]
             }
        },
        {
            "_id" : ObjectId("53f15b8beb75bdce84f914a4"),
            "value" : {
                "deviceId" : NumberLong("1377700226807"),
                "ts" : ISODate("2014-08-04T06:37:55Z"),
                "duration" : NumberLong(3600),
                "data" : [
                    {
                        "ts" : ISODate("2014-08-04T06:37:44Z"),
                        "loc" : {
                            "lon" : 5.1101176,
                            "lat" : 52.0625171
                        }
                    },
                    {
                        "ts" : ISODate("2014-08-04T06:37:38Z"),
                        "loc" : {
                            "lon" : 5.1101409,
                            "lat" : 52.0625818
                    },
                    {
                        "ts" : ISODate("2014-08-04T06:37:32Z"),
                        "loc" : {
                            "lon" : 5.1099513,
                            "lat" : 52.0624157
                        }
                    }
                ]
            }
        }
    ]

请注意，这里给出的输出是不同于样本的_id值，因为样本包含_id的"重复"值，这在主键上当然是不允许的。

相关内容

  • 没有找到相关文章

最新更新


热门标签：

javascript python java c# php android html jquery c++ css ios sql mysql arrays asp.net json python-3.x ruby-on-rails .net sql-server django objective-c excel regex ruby linux ajax iphone xml vba spring asp.net-mvc database wordpress string postgresql wpf windows xcode bash git oracle list vb.net multithreading eclipse algorithm macos powershell visual-studio image forms numpy scala function api selenium