Fix several issues reported by oss-fuzz (#3526)

- possible integer overflow in adjust_table_max_size:
  unsigned integer overflow: 2684354559 * 2 cannot be represented in type 'uint32'
- limit max memory size in wasm_runtime_malloc
- add more checks in aot loader
- adjust compilation options

core/iwasm/common/wasm_memory.c

@@ -284,6 +284,13 @@ wasm_runtime_malloc(unsigned int size)
 #endif
     }
 
+#if WASM_ENABLE_FUZZ_TEST != 0
+    if (size >= WASM_MEM_ALLOC_MAX_SIZE) {
+        LOG_WARNING("warning: wasm_runtime_malloc with too large size\n");
+        return NULL;
+    }
+#endif
+
     return wasm_runtime_malloc_internal(size);
 }