Verified Commit 206d6491 authored by AtjonTV's avatar AtjonTV
Browse files

Implement FFDB; Deprecate FileSync; And much more

* Check the Changelog for a complete list of everything that changed since 3.1.0
parent 7fe9ef37
Pipeline #2265 failed with stages
in 41 seconds
# Changelog
## Development
### Added
- kLib FFDB (FlatFile DataBase) Backend (will replace FileSync)
- `USE_FFDB_BACKEND` StoreFlag
- `writeToDisk(store,file)` - Required USE_FFDB_BACKEND Flag
- `readFromDisk(store,file)`
- `replaceFlags`
- `updateFlag`
### Changed
- Blob now implements `Serializable`
- StoreFlags now implements `Serializable`
- Version is now loaded from Manifest if possible
- Upgraded to kLib 5.0.0-dev.7
- Upgraded to Kotlin 1.3.70
- Upgraded to Gradle 6.3
### Deprecated
- FileStore Engine (Will be replaced by FFDB)
- `writeToCache`
- `readFromCache`
- `updateFlags` (Replaced with replaceFlags)
- `BlobPair` (Replaced with Pari<Blob, Blob>)
## Release 3.1.0
### Changed
......
......@@ -7,8 +7,15 @@ class Blob(
val uuid: String = UUID.randomUUID().toString(),
var data: Any,
var type: Class<*>
)
) : Serializable
@Deprecated(
message = "Use Pair<Blob, Blob> of Kotlin instead",
replaceWith = ReplaceWith(
expression = "Pair(blob1, blob2)"
),
level = DeprecationLevel.ERROR
)
class BlobPair(
var blob1: Blob?,
var blob2: Blob?
......@@ -28,4 +35,4 @@ fun blobOf(
fun blobPairOf(
blob1: Blob?,
blob2: Blob?
) = BlobPair(blob1, blob2)
\ No newline at end of file
) = Pair(blob1, blob2)
......@@ -46,4 +46,4 @@ infix fun List<Blob>.containsKey(key: String): Boolean {
infix fun List<Blob>.notContainsKey(key: String): Boolean {
key deny ""
return this.find { it.uuid == key } == null
}
\ No newline at end of file
}
package com.atvgstudios.cosnixdb
import java.io.File
import java.io.FileNotFoundException
import java.lang.Exception
import klib.SemVer
import klib.annotations.Experimental
import klib.extensions.asFile
import klib.extensions.deny
import klib.extensions.reset
import klib.extensions.toHex
import klib.ffdb.FFDB
import klib.functions.pairOf
import klib.text.KString
import kotlinx.coroutines.launch
import kotlinx.coroutines.runBlocking
import java.io.File
import java.lang.StringBuilder
object FaradayDB {
private val multiStore: MutableMap<String, MutableList<Blob>> = HashMap()
......@@ -18,7 +22,12 @@ object FaradayDB {
/**
* Get the FaradayDB Version
*/
val version = SemVer(major = 3, minor = 1, patch = 0)
val version: SemVer
get() {
val ver = javaClass.`package`.implementationVendor
return if (ver == null) SemVer(4, 0, 0, "unstable")
else SemVer.parse(ver)
}
/**
* Returns the Total Size of all stores
......@@ -69,17 +78,58 @@ object FaradayDB {
* @param flags The Flags for the Store
* @return True when override was OK, False when Store didnt exist
*/
@Deprecated("Replaced by the more appropriately named replaceFlags",
replaceWith = ReplaceWith("replaceFlags(store, flags)")
)
fun updateFlags(store: String, flags: List<StoreFlags>): Boolean {
store deny ""
if (containsStore(store)) {
multiFlag[store] = ArrayList()
multiFlag[store]!!.addAll(flags)
return true
saveStoreFlags(store)?.let {
it.clear()
it.addAll(flags)
}
return false
}
/**
* Overrides all Flags of a Store
*
* @param store The Store UUID
* @param flags The Flags for the Store
* @return True when override was OK, False when Store didnt exist
*/
fun replaceFlags(store: String, flags: List<StoreFlags>): Boolean {
store deny ""
saveStoreFlags(store)?.let {
it.clear()
it.addAll(flags)
}
return false
}
/**
* Add/Remove one Flag to a Store
*
* @param store The Store UUID
* @param flag The Flag to add/remove
* @return True when adding/removing was OK, False when Store didnt exist
*/
fun updateFlag(store: String, flag: StoreFlags): Boolean {
store deny ""
return saveStoreFlags(store)?.let<MutableList<StoreFlags>, Boolean> {
if (it.contains(flag)) {
it.remove(flag)
} else {
it.add(flag)
}
return true
} ?: false
}
/**
* Write a value into the Store.
*
......@@ -109,11 +159,7 @@ object FaradayDB {
fun all(store: String): List<Blob>? {
store deny ""
if (containsStore(store)) {
return multiStore[store]
}
return null
return saveStore(store)
}
/**
......@@ -140,10 +186,8 @@ object FaradayDB {
*/
fun contains(store: String, key: String): Boolean {
pairOf(store, key) deny ""
if (containsStore(store)) {
return multiStore[store]!!.containsKey(key)
}
return false
return saveStore(store)?.containsKey(key) ?: false
}
/**
......@@ -152,7 +196,7 @@ object FaradayDB {
* @param store The Store UUID
* @return True if Store exists
*/
fun containsStore(store: String): Boolean = multiStore.containsKey(store)
fun containsStore(store: String): Boolean = store in multiStore
/**
* Removes a Key from a Store
......@@ -163,10 +207,11 @@ object FaradayDB {
*/
fun remove(store: String, key: String): Boolean {
pairOf(store, key) deny ""
if (containsStore(store) && contains(store, key) && !multiFlag[store]!!.contains(StoreFlags.DENY_KEY_DELETE)) {
multiStore[store]!!.remove(get(store, key))
return true
if (containsStore(store) && contains(store, key) && !saveStoreFlags(store)!!.contains(StoreFlags.DENY_KEY_DELETE)) {
return saveStore(store)?.remove(get(store, key)) ?: false
}
return false
}
......@@ -178,10 +223,12 @@ object FaradayDB {
*/
fun clear(store: String): Boolean {
store deny ""
if (containsStore(store) && !multiFlag[store]!!.contains(StoreFlags.DENY_STORE_DELETE)) {
multiStore[store]!!.clear()
if (containsStore(store) && !saveStoreFlags(store)!!.contains(StoreFlags.DENY_STORE_DELETE)) {
saveStore(store)?.clear()
return true
}
return false
}
......@@ -189,7 +236,11 @@ object FaradayDB {
* Async write database to File System
*
* @param fileStore Path to write to
*
* @deprecated Replaced with writeToDisk since 4.0.0
*/
@Deprecated("This function uses the old FaradayDB file tree backend for on-disk storage",
replaceWith = ReplaceWith("writeToDisk"), level = DeprecationLevel.WARNING)
fun writeToCache(fileStore: String) = runBlocking {
val indexFile = File("$fileStore/index")
if (indexFile.exists()) indexFile.reset()
......@@ -205,12 +256,12 @@ object FaradayDB {
multiFlag.forEach { (uuid, flags) ->
launch {
val flagText = StringBuilder()
val flagText = KString()
for (i in 0 until flags.size) {
if (i == flags.size - 1) {
flagText.append("0x${flags[i].flag.toHex()}")
flagText += if (i == flags.size - 1) {
"0x${flags[i].flag.toHex()}"
} else {
flagText.append("0x${flags[i].flag.toHex()},")
"0x${flags[i].flag.toHex()},"
}
}
configFile.appendText("$uuid=[$flagText]\n")
......@@ -222,7 +273,12 @@ object FaradayDB {
* Async read database from File System
*
* @param fileStore Path to read from
*
* @deprecated Replaced with readFromDisk since 4.0.0
*/
@Deprecated("This function uses the old FaradayDB file tree backend for on-disk storage",
replaceWith = ReplaceWith("readFromDisk"), level = DeprecationLevel.WARNING
)
fun readFromCache(fileStore: String) = runBlocking {
FileSync.readStores(fileStore)
}
......@@ -232,8 +288,91 @@ object FaradayDB {
*
* @param fileStore Path to read from
* @param uuid UUID to read in
*
* @deprecated Replaced with readFromDisk since 4.0.0
*/
@Deprecated("This function uses the old FaradayDB file tree backend for on-disk storage",
replaceWith = ReplaceWith("readFromDisk"), level = DeprecationLevel.WARNING
)
fun readFromCache(fileStore: String, uuid: String) = runBlocking {
FileSync.readStore(fileStore, uuid)
}
/**
* Write data to Disk using the new FFDB Backend if enabled using USE_FFDB_BACKEND(0x01) flag
*
* @param store Store to write to disk
* @param file File to write into
*
* @since 4.0.0
* @author Thomas Obernosterer
*/
@OptIn(Experimental::class)
fun writeToDisk(store: String, file: String) {
pairOf(store, file) deny ""
// This storage feature is only available when the USE_FFDB_BACKEND flag is set on the Store
if (containsStore(store) && (saveStoreFlags(store)?.contains(StoreFlags.USE_FFDB_BACKEND) == true)) {
val db = FFDB.open(file)
db.writeAll(saveStoreFlags(store)!!)
db.writeAll(saveStore(store)!!)
db.flush()
} else {
throw Exception("Cannot use FFDB Backend without USE_FFDB_BACKEND(0x01) flag")
}
}
/**
* Read data from Disk using the new FFDB Backend
*
* Creates the store if it does not exist
*
* @param store Store to read into (Will be created if not exists)
* @param file File to read from
*
* @since 4.0.0
* @author Thomas Obernosterer
*/
@OptIn(Experimental::class)
fun readFromDisk(store: String, file: String) {
pairOf(store, file) deny ""
if (!containsStore(store)) {
// Create a new store and make sure to add the USE_FFDB_BACKEND flag
initStore(store, listOf(StoreFlags.USE_FFDB_BACKEND))
}
if (!file.asFile().exists()) {
throw FileNotFoundException()
}
val db = FFDB.open(file)
// We just read the complete dataset; We know that the first sets are flags, but we split them later
val data = db.readAll()
data.forEach {
when (it) {
// If the dataset is a Flag, add it to the flags
is StoreFlags -> updateFlag(store, it)
// If it is a Blob, add it to the data list
is Blob -> put(store, it)
}
}
}
private fun saveStore(store: String): MutableList<Blob>? {
if (containsStore(store)) {
return multiStore[store]
}
return null
}
private fun saveStoreFlags(store: String): MutableList<StoreFlags>? {
if (containsStore(store)) {
return multiFlag[store]
}
return null
}
}
package com.atvgstudios.cosnixdb
import klib.extensions.asFile
import java.io.File
import java.io.InvalidObjectException
import java.io.ObjectInputStream
import java.io.ObjectOutputStream
import klib.extensions.asFile
@Deprecated("Will be replaced with the kLib FFDB Backend (USE_FFDB_BACKEND(0x01))")
object FileSync {
suspend fun writeStore(fileStore: String, uuid: String, data: List<Blob>) {
val storeFolder = File("$fileStore/$uuid")
......@@ -63,14 +65,14 @@ object FileSync {
}
}
private suspend fun deserializeAndAppend(it: File, uuid: String) {
private fun deserializeAndAppend(it: File, uuid: String) {
val inputStream = ObjectInputStream(it.inputStream())
val data = inputStream.readObject()
if (data is OnDiscBlob) {
FaradayDB.put(uuid, Blob(uuid = uuid, data = data.data, type = data.type))
} else {
// TODO: FAIL HERE
throw InvalidObjectException("Data from Disc was not in a readable format!")
}
}
}
\ No newline at end of file
}
package com.atvgstudios.cosnixdb
import java.io.Serializable
/*
SYSTEM Flags begin with 0x0
KEY Flags begin with 0x1
STORE Flags begin with 0x2
*/
enum class StoreFlags(val flag: Byte) {
enum class StoreFlags(val flag: Byte) : Serializable {
USE_FFDB_BACKEND(0x01),
ALLOW_KEY_OVERRIDE(0x10),
DENY_KEY_DELETE(0x11),
DENY_STORE_DELETE(0x20);
......@@ -19,8 +22,9 @@ fun storeFlagsOf(vararg bytes: Byte): List<StoreFlags> {
StoreFlags.ALLOW_KEY_OVERRIDE.ordinal -> list.add(StoreFlags.ALLOW_KEY_OVERRIDE)
StoreFlags.DENY_KEY_DELETE.ordinal -> list.add(StoreFlags.DENY_KEY_DELETE)
StoreFlags.DENY_STORE_DELETE.ordinal -> list.add(StoreFlags.DENY_STORE_DELETE)
StoreFlags.USE_FFDB_BACKEND.ordinal -> list.add(StoreFlags.USE_FFDB_BACKEND)
}
}
return list
}
\ No newline at end of file
}
import com.atvgstudios.cosnixdb.*
import com.atvgstudios.cosnixdb.Blob
import com.atvgstudios.cosnixdb.FaradayDB
import com.atvgstudios.cosnixdb.StoreFlags
import klib.SemVer
import klib.uuid.UniqueID
import kotlin.random.Random
import kotlin.system.measureTimeMillis
val uuid = UniqueID.random
val dbFile = "/stmp/test.ffdb"
fun testWrite() {
val data: MutableList<Blob> = ArrayList()
for(i in 0 until 10_000) {
for (i in 0 until 10_000) {
data.add(Blob(data = Random.nextInt(), type = Integer::class.java))
}
FaradayDB.initStore(uuid, listOf(StoreFlags.DENY_STORE_DELETE, StoreFlags.DENY_KEY_DELETE))
FaradayDB.initStore(uuid, listOf(StoreFlags.DENY_STORE_DELETE, StoreFlags.DENY_KEY_DELETE, StoreFlags.USE_FFDB_BACKEND))
data.forEach {
FaradayDB.put(uuid, it)
......@@ -20,46 +23,25 @@ fun testWrite() {
FaradayDB.put(uuid, Blob(uuid = "0", data = "Hello", type = String::class.java))
FaradayDB.writeToCache("/tmp/test.dbs")
println("Write to Disk using Faraday FileTree took (ms): " + measureTimeMillis {
FaradayDB.writeToCache("/stmp/testing.db")
})
println("Write to Disk using FFDB took (ms): " + measureTimeMillis {
FaradayDB.writeToDisk(uuid, dbFile)
})
}
fun testRead() {
FaradayDB.readFromDisk(uuid, dbFile)
println(FaradayDB.stackSize)
println(FaradayDB.all(uuid)!!.last().data)
}
//
//fun testRead() {
// val fileStore = File("/tmp/test.dbs")
//
// val storeFolders = fileStore.listFiles()!!
//
// storeFolders.forEach { storeFolder ->
// if(storeFolder.isFile || storeFolder.listFiles() == null || storeFolder.listFiles()!!.isEmpty()) return@forEach
// val storeFiles = storeFolder.listFiles()!!
//
// FaradayDB.initStore(storeFolder.name, storeFlagsOf())
//
// println(storeFolder.name)
//
// storeFiles.forEach {
// val name = it.name.removeSuffix(".bin")
//
// val inputStream = ObjectInputStream(it.inputStream())
// val data = inputStream.readObject()
//
// println(" ID: $name; Data: $data")
//
// FaradayDB.put(storeFolder.name, Blob(name, data))
// }
// }
//}
fun main() {
if (FaradayDB.version < SemVer(3,0,0)) {
println("Cannot use FaradayDB ${FaradayDB.version}. Version 3.0.0 is required!")
if (FaradayDB.version < SemVer(4, 0, 0, "unstable")) {
println("Cannot use FaradayDB ${FaradayDB.version}. Version 4.0.0 is required!")
return
}
testWrite()
FaradayDB.readFromCache("/tmp/test.dbs")
// FaradayDB.writeToCache("/tmp/test.dbs")
// testRead()
println(FaradayDB.stackSize)
// val strings = (FaradayDB.all(uuid) ?: ArrayList()) findByType String::class.java
// println(strings.size)
}
\ No newline at end of file
testRead()
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment