如何在dynamodb中超过1mb的扫描数据限制
问题描述:
我正在使用dynamodb和nodejs,我有3000条记录,并且我在代码中编写了60多个段,每段扫描1mb数据并显示结果到60多个1MB的限制。因此,请提供解决方案,如何在单个步骤中获得3000条记录扫描,这意味着在一个分段中。请迅速提供解决方案,因为我在项目中间受到了攻击。请帮帮我。以下是我的代码。如何在dynamodb中超过1mb的扫描数据限制
var AWS = require("aws-sdk");
var async = require("async");
AWS.config.update({
region: "---",
endpoint: "-----------",
accessKeyId: "-----------------",
secretAccessKey:"----------"
});
var db = new AWS.DynamoDB.DocumentClient()
var table = "rets_property_all";
var pstart =new Date() .getTime();
async.parallel({
0 : function(callback){
db.scan ({TableName: table,
ProjectionExpression: "#cityname,ListingKey ",
FilterExpression: "#cityname = :v_id",
ExpressionAttributeNames: {
"#cityname": "CityName",
},
ExpressionAttributeValues: {":v_id" : 'BALTIMORE'},
TotalSegments: 63,
Segment: 0//by the worker who has been called
},function (err , res) {
callback (null , res.Items);
});
},
1 : function(callback){
db.scan ({TableName: table,
ProjectionExpression: "#cityname,ListingKey ",
FilterExpression: "#cityname = :v_id",
ExpressionAttributeNames: {
"#cityname": "CityName",
},
ExpressionAttributeValues: {":v_id" : 'BALTIMORE'},
TotalSegments: 63,
Segment: 1//by the worker who has been called
}, function (err , res) {
callback (null , res.Items);
});
},
2 : function(callback){
db.scan ({TableName: table,
ProjectionExpression: "#cityname,ListingKey ",
FilterExpression: "#cityname = :v_id",
ExpressionAttributeNames: {
"#cityname": "CityName",
},
ExpressionAttributeValues: {":v_id" : 'BALTIMORE'},
TotalSegments: 63,
Segment: 2//by the worker who has been called
}, function (err , res) {
callback (null , res.Items);
});
},
--------
---------
------
62 : function(callback){
db.scan ({TableName: table,
ProjectionExpression: "#cityname,ListingKey ",
FilterExpression: "#cityname = :v_id",
ExpressionAttributeNames: {
"#cityname": "CityName",
},
ExpressionAttributeValues: {":v_id" : 'BALTIMORE'},
TotalSegments: 63,
Segment: 62//by the worker who has been called
}, function (err , res) {
callback (null , res.Items);
});
},
},function(err,results){
if (err) {throw err; }
var pend = new Date() .getTime();
console.log (results);
})
答
其实,有没有办法覆盖扫描1 MB限制。这是DynamoDB设计限制,不能被任何API覆盖。您可能需要了解体系结构或AWS服务设计的限制。
您可以在随后的扫描中使用LastEvaluatedKey
从上次扫描结束时开始。
扫描的结果集限于每次调用1 MB。您可以使用扫描响应中的LastEvaluatedKey 来检索更多结果。
用例不清楚为什么你想在一次扫描中获得所有3000条记录。即使您有特定用例,也无法在DynamoDB扫描中实现。
即使在关系型数据库中,您也可以获取游标并遍历迭代以顺序获取所有记录。同样,在DynamoDB中,您必须递归使用Scan,直到LastEvaluatedKey
为空。单扫描
示例代码递归直到LastEvaluatedKey为空: -
var docClient = new AWS.DynamoDB.DocumentClient();
var params = {
TableName: table,
ProjectionExpression: "#cityname,ListingKey ",
FilterExpression: "#cityname = :v_id",
ExpressionAttributeNames: {
"#cityname": "CityName",
},
ExpressionAttributeValues: { ":v_id": 'BALTIMORE' }
};
docClient.scan(params, onScan);
function onScan(err, data) {
if (err) {
console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
} else {
// print all the movies
console.log("Scan succeeded.");
data.Items.forEach(function (movie) {
console.log("Item :", JSON.stringify(movie));
});
// continue scanning if we have more movies
if (typeof data.LastEvaluatedKey != "undefined") {
console.log("Scanning for more...");
params.ExclusiveStartKey = data.LastEvaluatedKey;
docClient.scan(params, onScan);
}
}
}
我希望所有的3000条记录在一个扫描由于性能问题。它需要4.5秒。为了改善这一点,有任何解决方案。 – purushottam
如果上述代码有任何更正,请提供正确的代码。 – purushottam