mirror of
				https://github.com/0glabs/0g-chain.git
				synced 2025-10-31 15:17:27 +00:00 
			
		
		
		
	Make read-async-io configurable (#1732)
* Make read-async-io configurable * Added unit-test for read options configuration
This commit is contained in:
		
							parent
							
								
									9aefbac0e8
								
							
						
					
					
						commit
						0598b99063
					
				| @ -74,6 +74,8 @@ const ( | |||||||
| 	cacheIndexAndFilterBlocksBBTOOptName        = "rocksdb.cache_index_and_filter_blocks" | 	cacheIndexAndFilterBlocksBBTOOptName        = "rocksdb.cache_index_and_filter_blocks" | ||||||
| 	pinL0FilterAndIndexBlocksInCacheBBTOOptName = "rocksdb.pin_l0_filter_and_index_blocks_in_cache" | 	pinL0FilterAndIndexBlocksInCacheBBTOOptName = "rocksdb.pin_l0_filter_and_index_blocks_in_cache" | ||||||
| 	formatVersionBBTOOptName                    = "rocksdb.format_version" | 	formatVersionBBTOOptName                    = "rocksdb.format_version" | ||||||
|  | 
 | ||||||
|  | 	asyncIOReadOptName = "rocksdb.read-async-io" | ||||||
| ) | ) | ||||||
| 
 | 
 | ||||||
| func OpenDB(appOpts types.AppOptions, home string, backendType dbm.BackendType) (dbm.DB, error) { | func OpenDB(appOpts types.AppOptions, home string, backendType dbm.BackendType) (dbm.DB, error) { | ||||||
| @ -98,6 +100,7 @@ func openRocksdb(dir string, appOpts types.AppOptions) (dbm.DB, error) { | |||||||
| 	cfOpts.SetBlockBasedTableFactory(bbtoOpts) | 	cfOpts.SetBlockBasedTableFactory(bbtoOpts) | ||||||
| 	dbOpts = overrideDBOpts(dbOpts, appOpts) | 	dbOpts = overrideDBOpts(dbOpts, appOpts) | ||||||
| 	cfOpts = overrideCFOpts(cfOpts, appOpts) | 	cfOpts = overrideCFOpts(cfOpts, appOpts) | ||||||
|  | 	readOpts := readOptsFromAppOpts(appOpts) | ||||||
| 
 | 
 | ||||||
| 	enableMetrics := cast.ToBool(appOpts.Get(enableMetricsOptName)) | 	enableMetrics := cast.ToBool(appOpts.Get(enableMetricsOptName)) | ||||||
| 	reportMetricsIntervalSecs := cast.ToInt64(appOpts.Get(reportMetricsIntervalSecsOptName)) | 	reportMetricsIntervalSecs := cast.ToInt64(appOpts.Get(reportMetricsIntervalSecsOptName)) | ||||||
| @ -105,7 +108,7 @@ func openRocksdb(dir string, appOpts types.AppOptions) (dbm.DB, error) { | |||||||
| 		reportMetricsIntervalSecs = defaultReportMetricsIntervalSecs | 		reportMetricsIntervalSecs = defaultReportMetricsIntervalSecs | ||||||
| 	} | 	} | ||||||
| 
 | 
 | ||||||
| 	return newRocksDBWithOptions("application", dir, dbOpts, cfOpts, enableMetrics, reportMetricsIntervalSecs) | 	return newRocksDBWithOptions("application", dir, dbOpts, cfOpts, readOpts, enableMetrics, reportMetricsIntervalSecs) | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| // loadLatestOptions loads and returns database and column family options
 | // loadLatestOptions loads and returns database and column family options
 | ||||||
| @ -237,6 +240,16 @@ func overrideCFOpts(cfOpts *grocksdb.Options, appOpts types.AppOptions) *grocksd | |||||||
| 	return cfOpts | 	return cfOpts | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
|  | func readOptsFromAppOpts(appOpts types.AppOptions) *grocksdb.ReadOptions { | ||||||
|  | 	ro := grocksdb.NewDefaultReadOptions() | ||||||
|  | 	asyncIO := appOpts.Get(asyncIOReadOptName) | ||||||
|  | 	if asyncIO != nil { | ||||||
|  | 		ro.SetAsyncIO(cast.ToBool(asyncIO)) | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	return ro | ||||||
|  | } | ||||||
|  | 
 | ||||||
| func bbtoFromAppOpts(appOpts types.AppOptions) *grocksdb.BlockBasedTableOptions { | func bbtoFromAppOpts(appOpts types.AppOptions) *grocksdb.BlockBasedTableOptions { | ||||||
| 	bbto := defaultBBTO() | 	bbto := defaultBBTO() | ||||||
| 
 | 
 | ||||||
| @ -282,6 +295,7 @@ func newRocksDBWithOptions( | |||||||
| 	dir string, | 	dir string, | ||||||
| 	dbOpts *grocksdb.Options, | 	dbOpts *grocksdb.Options, | ||||||
| 	cfOpts *grocksdb.Options, | 	cfOpts *grocksdb.Options, | ||||||
|  | 	readOpts *grocksdb.ReadOptions, | ||||||
| 	enableMetrics bool, | 	enableMetrics bool, | ||||||
| 	reportMetricsIntervalSecs int64, | 	reportMetricsIntervalSecs int64, | ||||||
| ) (*dbm.RocksDB, error) { | ) (*dbm.RocksDB, error) { | ||||||
| @ -307,11 +321,10 @@ func newRocksDBWithOptions( | |||||||
| 		go reportMetrics(db, time.Second*time.Duration(reportMetricsIntervalSecs)) | 		go reportMetrics(db, time.Second*time.Duration(reportMetricsIntervalSecs)) | ||||||
| 	} | 	} | ||||||
| 
 | 
 | ||||||
| 	ro := grocksdb.NewDefaultReadOptions() |  | ||||||
| 	wo := grocksdb.NewDefaultWriteOptions() | 	wo := grocksdb.NewDefaultWriteOptions() | ||||||
| 	woSync := grocksdb.NewDefaultWriteOptions() | 	woSync := grocksdb.NewDefaultWriteOptions() | ||||||
| 	woSync.SetSync(true) | 	woSync.SetSync(true) | ||||||
| 	return dbm.NewRocksDBWithRawDB(db, ro, wo, woSync), nil | 	return dbm.NewRocksDBWithRawDB(db, readOpts, wo, woSync), nil | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| // newDefaultOptions returns default tm-db options for RocksDB, see for details:
 | // newDefaultOptions returns default tm-db options for RocksDB, see for details:
 | ||||||
|  | |||||||
| @ -186,7 +186,7 @@ func TestLoadLatestOptions(t *testing.T) { | |||||||
| 					require.NoError(t, err) | 					require.NoError(t, err) | ||||||
| 				}() | 				}() | ||||||
| 
 | 
 | ||||||
| 				db, err := newRocksDBWithOptions(name, dir, tc.dbOpts, tc.cfOpts, true, defaultReportMetricsIntervalSecs) | 				db, err := newRocksDBWithOptions(name, dir, tc.dbOpts, tc.cfOpts, grocksdb.NewDefaultReadOptions(), true, defaultReportMetricsIntervalSecs) | ||||||
| 				require.NoError(t, err) | 				require.NoError(t, err) | ||||||
| 				require.NoError(t, db.Close()) | 				require.NoError(t, db.Close()) | ||||||
| 
 | 
 | ||||||
| @ -321,6 +321,33 @@ func TestOverrideCFOpts(t *testing.T) { | |||||||
| 	} | 	} | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
|  | func TestReadOptsFromAppOpts(t *testing.T) { | ||||||
|  | 	for _, tc := range []struct { | ||||||
|  | 		desc           string | ||||||
|  | 		mockAppOptions *mockAppOptions | ||||||
|  | 		asyncIO        bool | ||||||
|  | 	}{ | ||||||
|  | 		{ | ||||||
|  | 			desc:           "default options", | ||||||
|  | 			mockAppOptions: newMockAppOptions(map[string]interface{}{}), | ||||||
|  | 			asyncIO:        false, | ||||||
|  | 		}, | ||||||
|  | 		{ | ||||||
|  | 			desc: "set asyncIO option to true", | ||||||
|  | 			mockAppOptions: newMockAppOptions(map[string]interface{}{ | ||||||
|  | 				asyncIOReadOptName: true, | ||||||
|  | 			}), | ||||||
|  | 			asyncIO: true, | ||||||
|  | 		}, | ||||||
|  | 	} { | ||||||
|  | 		t.Run(tc.desc, func(t *testing.T) { | ||||||
|  | 			readOpts := readOptsFromAppOpts(tc.mockAppOptions) | ||||||
|  | 
 | ||||||
|  | 			require.Equal(t, tc.asyncIO, readOpts.IsAsyncIO()) | ||||||
|  | 		}) | ||||||
|  | 	} | ||||||
|  | } | ||||||
|  | 
 | ||||||
| func TestNewRocksDBWithOptions(t *testing.T) { | func TestNewRocksDBWithOptions(t *testing.T) { | ||||||
| 	defaultOpts := newDefaultOptions() | 	defaultOpts := newDefaultOptions() | ||||||
| 
 | 
 | ||||||
| @ -337,7 +364,7 @@ func TestNewRocksDBWithOptions(t *testing.T) { | |||||||
| 	cfOpts := newDefaultOptions() | 	cfOpts := newDefaultOptions() | ||||||
| 	cfOpts.SetWriteBufferSize(999_999) | 	cfOpts.SetWriteBufferSize(999_999) | ||||||
| 
 | 
 | ||||||
| 	db, err := newRocksDBWithOptions(name, dir, dbOpts, cfOpts, true, defaultReportMetricsIntervalSecs) | 	db, err := newRocksDBWithOptions(name, dir, dbOpts, cfOpts, grocksdb.NewDefaultReadOptions(), true, defaultReportMetricsIntervalSecs) | ||||||
| 	require.NoError(t, err) | 	require.NoError(t, err) | ||||||
| 	require.NoError(t, db.Close()) | 	require.NoError(t, db.Close()) | ||||||
| 
 | 
 | ||||||
|  | |||||||
		Loading…
	
		Reference in New Issue
	
	Block a user
	 Evgeniy Scherbina
						Evgeniy Scherbina