Improve handling of exceptions which should trigger retrying an operation

src/database/utils/nested-transactions.ts

@@ -0,0 +1,62 @@
+/*
+ * server component for the TimeLimit App
+ * Copyright (C) 2019 - 2021 Jonas Lochmann
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { generateIdWithinFamily } from '../../util/token'
+import { configItemIds } from '../config'
+import { Database } from '../main'
+
+class NestedTransactionTestException extends Error {}
+class TestRollbackException extends NestedTransactionTestException {}
+class NestedTransactionsNotWorkingException extends NestedTransactionTestException { constructor () { super('NestedTransactionsNotWorkingException') } }
+class IllegalStateException extends NestedTransactionTestException {}
+
+export async function assertNestedTransactionsAreWorking (database: Database) {
+  const testValue = generateIdWithinFamily()
+
+  // clean up just for the case
+  await database.config.destroy({ where: { id: configItemIds.selfTestData } })
+
+  await database.transaction(async (transaction) => {
+    const readOne = await database.config.findOne({ where: { id: configItemIds.selfTestData }, transaction })
+
+    if (readOne) throw new IllegalStateException()
+
+    await database.transaction(async (transaction) => {
+      await database.config.create({ id: configItemIds.selfTestData, value: testValue }, { transaction })
+
+      const readTwo = await database.config.findOne({ where: { id: configItemIds.selfTestData }, transaction })
+
+      if (readTwo?.value !== testValue) throw new IllegalStateException()
+
+      try {
+        await database.transaction(async (transaction) => {
+          await database.config.destroy({ where: { id: configItemIds.selfTestData }, transaction })
+
+          throw new TestRollbackException()
+        }, { transaction })
+      } catch (ex) {
+        if (!(ex instanceof TestRollbackException)) throw ex
+      }
+
+      const readThree = await database.config.findOne({ where: { id: configItemIds.selfTestData }, transaction })
+
+      if (readThree?.value !== testValue) throw new NestedTransactionsNotWorkingException()
+
+      await database.config.destroy({ where: { id: configItemIds.selfTestData }, transaction })
+    }, { transaction })
+  })
+}

src/database/utils/serialized.ts

@@ -0,0 +1,106 @@
+/*
+ * server component for the TimeLimit App
+ * Copyright (C) 2019 - 2021 Jonas Lochmann
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import * as Sequelize from 'sequelize'
+import { configItemIds } from '../config'
+import { Database } from '../main'
+
+export class SerializationFeatureCheckException extends Error {}
+
+export function shouldRetryWithException(database: Database, e: any): boolean {
+  if (e instanceof Sequelize.TimeoutError) return true
+
+  if (!(e instanceof Sequelize.DatabaseError)) return false
+
+  const parent = e.parent
+
+  if (typeof parent !== 'object') return false
+
+  if (database.dialect === 'sqlite') {
+    if (parent.message.startsWith('SQLITE_BUSY:')) return true
+  } else if (database.dialect === 'postgres') {
+    // 40001 = serialization_failure
+    if ((parent as any).code === '40001') return true
+    // 40P01 = deadlock detected
+    if ((parent as any).code === '40P01') return true
+  } else if (database.dialect === 'mariadb') {
+    const errno = (parent as any).errno
+
+    // ER_LOCK_DEADLOCK
+    // Deadlock found when trying to get lock; try restarting transaction
+    if (errno === 1213) return true
+  }
+
+  return false
+}
+
+export async function assertSerializeableTransactionsAreWorking(database: Database) {
+  // clean up just for the case
+  await database.config.destroy({
+    where: {
+      id: {
+        [Sequelize.Op.in]: [ configItemIds.selfTestData, configItemIds.secondSelfTestData ]
+      }
+    }
+  })
+
+  // insert specific data
+  await database.config.bulkCreate([
+    {
+      id: configItemIds.selfTestData,
+      value: '123'
+    },
+    {
+      id: configItemIds.secondSelfTestData,
+      value: '456'
+    }
+  ])
+
+  try {
+    // use two parallel transactions
+    await database.transaction(async (transactionOne) => {
+      await database.transaction(async (transactionTwo) => {
+        await database.config.findAll({ transaction: transactionOne })
+        await database.config.findAll({ transaction: transactionTwo })
+
+        await Promise.all([
+          (async () => {
+            await database.config.update({ value: 'c' }, { where: { id: configItemIds.selfTestData }, transaction: transactionOne })
+          })(),
+          (async () => {
+            await database.config.update({ value: 'd' }, { where: { id: configItemIds.secondSelfTestData }, transaction: transactionTwo })
+          })(),
+        ])
+      })
+    })
+
+    throw new SerializationFeatureCheckException()
+  } catch (ex) {
+    if (!shouldRetryWithException(database, ex)) {
+      throw new SerializationFeatureCheckException()
+    }
+  }
+
+  // finish clean up
+  await database.config.destroy({
+    where: {
+      id: {
+        [Sequelize.Op.in]: [ configItemIds.selfTestData, configItemIds.secondSelfTestData ]
+      }
+    }
+  })
+}