时间:2021-07-01 10:21:17 帮助过:16人阅读
1 #ifdef USE_LMDB 2 #include "caffe/util/db_lmdb.hpp" 3 4 #include <sys/stat.h> 5 6 #include <string> 7 8 namespace caffe { namespace db { 9 10 void LMDB::Open(const string& source, Mode mode) { 11 MDB_CHECK(mdb_env_create(&mdb_env_)); 12 if (mode == NEW) { 13 CHECK_EQ(mkdir(source.c_str(), 0744), 0) << "mkdir " << source << " failed"; //如果为NEW, 则创建一个source的路径; 14 } 15 int flags = 0; 16 if (mode == READ) { 17 flags = MDB_RDONLY | MDB_NOTLS; //设置一下它的特别选项,一个是只读,一个是不使用线程本地存储; 18 } 19 int rc = mdb_env_open(mdb_env_, source.c_str(), flags, 0664); 20 #ifndef ALLOW_LMDB_NOLOCK 21 MDB_CHECK(rc); 22 #else 23 if (rc == EACCES) { //表示:the user didn‘t have permission to access the environment files. 24 LOG(WARNING) << "Permission denied. Trying with MDB_NOLOCK ..."; 25 // Close and re-open environment handle 26 mdb_env_close(mdb_env_); 27 MDB_CHECK(mdb_env_create(&mdb_env_)); 28 // Try again with MDB_NOLOCK 29 flags |= MDB_NOLOCK; //增加了一个选项,它的意思为不作任何锁操作; 30 MDB_CHECK(mdb_env_open(mdb_env_, source.c_str(), flags, 0664)); 31 } else { 32 MDB_CHECK(rc); 33 } 34 #endif 35 LOG(INFO) << "Opened lmdb " << source; 36 } 37 38 LMDBCursor* LMDB::NewCursor() { 39 MDB_txn* mdb_txn; 40 MDB_cursor* mdb_cursor; 41 MDB_CHECK(mdb_txn_begin(mdb_env_, NULL, MDB_RDONLY, &mdb_txn)); 42 MDB_CHECK(mdb_dbi_open(mdb_txn, NULL, 0, &mdb_dbi_)); 43 MDB_CHECK(mdb_cursor_open(mdb_txn, mdb_dbi_, &mdb_cursor)); 44 return new LMDBCursor(mdb_txn, mdb_cursor); //不明白为什么把局部的mdb_txn返回去,在LMDBCursor类里面,也就析构函数用到了mdb_txn, 45 //可以就是为了去用函数mdb_txn_abort把它干掉吧;,似乎明白了呢,等用完它,把它干掉吧; 46 } 47 48 LMDBTransaction* LMDB::NewTransaction() { 49 return new LMDBTransaction(mdb_env_); 50 } 51 52 void LMDBTransaction::Put(const string& key, const string& value) { 53 keys.push_back(key); 54 values.push_back(value); 55 } 56 57 void LMDBTransaction::Commit() { 58 MDB_dbi mdb_dbi; 59 MDB_val mdb_key, mdb_data; 60 MDB_txn *mdb_txn; 61 62 // Initialize MDB variables 63 MDB_CHECK(mdb_txn_begin(mdb_env_, NULL, 0, &mdb_txn)); 64 MDB_CHECK(mdb_dbi_open(mdb_txn, NULL, 0, &mdb_dbi)); 65 66 for (int i = 0; i < keys.size(); i++) { 67 mdb_key.mv_size = keys[i].size(); 68 mdb_key.mv_data = const_cast<char*>(keys[i].data()); 69 mdb_data.mv_size = values[i].size(); 70 mdb_data.mv_data = const_cast<char*>(values[i].data()); 71 72 // Add data to the transaction 73 int put_rc = mdb_put(mdb_txn, mdb_dbi, &mdb_key, &mdb_data, 0); //即把key/data的值放入database里去; 74 if (put_rc == MDB_MAP_FULL) { 75 // Out of memory - double the map size and retry 76 mdb_txn_abort(mdb_txn); 77 mdb_dbi_close(mdb_env_, mdb_dbi); 78 DoubleMapSize(); 79 Commit(); //不明白 为什么能调用 Commit(),回去复习一下C++; 80 return; 81 } 82 // May have failed for some other reason 83 MDB_CHECK(put_rc); 84 } 85 86 // Commit the transaction 87 int commit_rc = mdb_txn_commit(mdb_txn); 88 if (commit_rc == MDB_MAP_FULL) { 89 // Out of memory - double the map size and retry 90 mdb_dbi_close(mdb_env_, mdb_dbi); 91 DoubleMapSize(); 92 Commit(); 93 return; 94 } 95 // May have failed for some other reason 96 MDB_CHECK(commit_rc); 97 98 // Cleanup after successful commit 99 mdb_dbi_close(mdb_env_, mdb_dbi); 100 keys.clear(); //清空keys and values; 101 values.clear(); 102 } 103 104 void LMDBTransaction::DoubleMapSize() { 105 struct MDB_envinfo current_info; 106 MDB_CHECK(mdb_env_info(mdb_env_, ¤t_info)); 107 size_t new_size = current_info.me_mapsize * 2; 108 DLOG(INFO) << "Doubling LMDB map size to " << (new_size>>20) << "MB ..."; 109 MDB_CHECK(mdb_env_set_mapsize(mdb_env_, new_size)); 110 } 111 112 } // namespace db 113 } // namespace caffe 114 #endif // USE_LMDB
caffe的db_lmdb.hpp文件
标签:char files key clear nbsp class failed 入参 div