Source code
Revision control
Copy as Markdown
Other Tools
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
#include "mozilla/dom/cache/DBSchema.h"
#include "ipc/IPCMessageUtils.h"
#include "mozIStorageConnection.h"
#include "mozIStorageFunction.h"
#include "mozIStorageStatement.h"
#include "mozStorageHelper.h"
#include "mozilla/BasePrincipal.h"
#include "mozilla/ResultExtensions.h"
#include "mozilla/StaticPrefs_extensions.h"
#include "mozilla/dom/HeadersBinding.h"
#include "mozilla/dom/InternalHeaders.h"
#include "mozilla/dom/InternalResponse.h"
#include "mozilla/dom/RequestBinding.h"
#include "mozilla/dom/ResponseBinding.h"
#include "mozilla/dom/cache/CacheCommon.h"
#include "mozilla/dom/cache/CacheTypes.h"
#include "mozilla/dom/cache/FileUtils.h"
#include "mozilla/dom/cache/SavedTypes.h"
#include "mozilla/dom/cache/TypeUtils.h"
#include "mozilla/dom/cache/Types.h"
#include "mozilla/dom/quota/ResultExtensions.h"
#include "mozilla/psm/TransportSecurityInfo.h"
#include "mozilla/storage/Variant.h"
#include "nsCOMPtr.h"
#include "nsCharSeparatedTokenizer.h"
#include "nsComponentManagerUtils.h"
#include "nsHttp.h"
#include "nsIContentPolicy.h"
#include "nsICryptoHash.h"
#include "nsIURI.h"
#include "nsNetCID.h"
#include "nsPrintfCString.h"
#include "nsTArray.h"
namespace mozilla::dom::cache::db {
const int32_t kFirstShippedSchemaVersion = 15;
namespace {
// ## Firefox 57 Cache API v25/v26/v27 Schema Hack Info
// ### Overview
// In Firefox 57 we introduced Cache API schema version 26 and Quota Manager
// schema v3 to support tracking padding for opaque responses. Unfortunately,
// Firefox 57 is a big release that may potentially result in users downgrading
// to Firefox 56 due to 57 retiring add-ons. These schema changes have the
// unfortunate side-effect of causing QuotaManager and all its clients to break
// if the user downgrades to 56. In order to avoid making a bad situation
// worse, we're now retrofitting 57 so that Firefox 56 won't freak out.
//
// ### Implementation
// We're introducing a new schema version 27 that uses an on-disk schema version
// of v25. We differentiate v25 from v27 by the presence of the column added
// by v26. This translates to:
// - v25: on-disk schema=25, no "response_padding_size" column in table
// "entries".
// - v26: on-disk schema=26, yes "response_padding_size" column in table
// "entries".
// - v27: on-disk schema=25, yes "response_padding_size" column in table
// "entries".
//
// ### Fallout
// Firefox 57 is happy because it sees schema 27 and everything is as it
// expects.
//
// Firefox 56 non-DEBUG build is fine/happy, but DEBUG builds will not be.
// - Our QuotaClient will invoke `NS_WARNING("Unknown Cache file found!");`
// at QuotaManager init time. This is harmless but annoying and potentially
// misleading.
// - The DEBUG-only Validate() call will error out whenever an attempt is made
// to open a DOM Cache database because it will notice the schema is broken
// and there is no attempt at recovery.
//
const int32_t kHackyDowngradeSchemaVersion = 25;
const int32_t kHackyPaddingSizePresentVersion = 27;
//
// Update this whenever the DB schema is changed.
const int32_t kLatestSchemaVersion = 29;
// ---------
// The following constants define the SQL schema. These are defined in the
// same order the SQL should be executed in CreateOrMigrateSchema(). They are
// broken out as constants for convenient use in validation and migration.
// ---------
// The caches table is the single source of truth about what Cache
// objects exist for the origin. The contents of the Cache are stored
// in the entries table that references back to caches.
//
// The caches table is also referenced from storage. Rows in storage
// represent named Cache objects. There are cases, however, where
// a Cache can still exist, but not be in a named Storage. For example,
// when content is still using the Cache after CacheStorage::Delete()
// has been run.
//
// For now, the caches table mainly exists for data integrity with
// foreign keys, but could be expanded to contain additional cache object
// information.
//
// AUTOINCREMENT is necessary to prevent CacheId values from being reused.
const char kTableCaches[] =
"CREATE TABLE caches ("
"id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT "
")";
// Security blobs are quite large and duplicated for every Response from
// the same https origin. This table is used to de-duplicate this data.
const char kTableSecurityInfo[] =
"CREATE TABLE security_info ("
"id INTEGER NOT NULL PRIMARY KEY, "
"hash BLOB NOT NULL, " // first 8-bytes of the sha1 hash of data column
"data BLOB NOT NULL, " // full security info data, usually a few KB
"refcount INTEGER NOT NULL"
")";
// Index the smaller hash value instead of the large security data blob.
const char kIndexSecurityInfoHash[] =
"CREATE INDEX security_info_hash_index ON security_info (hash)";
const char kTableEntries[] =
"CREATE TABLE entries ("
"id INTEGER NOT NULL PRIMARY KEY, "
"request_method TEXT NOT NULL, "
"request_url_no_query TEXT NOT NULL, "
"request_url_no_query_hash BLOB NOT NULL, " // first 8-bytes of sha1 hash
"request_url_query TEXT NOT NULL, "
"request_url_query_hash BLOB NOT NULL, " // first 8-bytes of sha1 hash
"request_referrer TEXT NOT NULL, "
"request_headers_guard INTEGER NOT NULL, "
"request_mode INTEGER NOT NULL, "
"request_credentials INTEGER NOT NULL, "
"request_contentpolicytype INTEGER NOT NULL, "
"request_cache INTEGER NOT NULL, "
"request_body_id TEXT NULL, "
"response_type INTEGER NOT NULL, "
"response_status INTEGER NOT NULL, "
"response_status_text TEXT NOT NULL, "
"response_headers_guard INTEGER NOT NULL, "
"response_body_id TEXT NULL, "
"response_security_info_id INTEGER NULL REFERENCES security_info(id), "
"response_principal_info TEXT NOT NULL, "
"cache_id INTEGER NOT NULL REFERENCES caches(id) ON DELETE CASCADE, "
"request_redirect INTEGER NOT NULL, "
"request_referrer_policy INTEGER NOT NULL, "
"request_integrity TEXT NOT NULL, "
"request_url_fragment TEXT NOT NULL, "
"response_padding_size INTEGER NULL, "
"request_body_disk_size INTEGER NULL, "
"response_body_disk_size INTEGER NULL "
// New columns must be added at the end of table to migrate and
// validate properly.
")";
// Create an index to support the QueryCache() matching algorithm. This
// needs to quickly find entries in a given Cache that match the request
// URL. The url query is separated in order to support the ignoreSearch
// option. Finally, we index hashes of the URL values instead of the
// actual strings to avoid excessive disk bloat. The index will duplicate
// the contents of the columsn in the index. The hash index will prune
// the vast majority of values from the query result so that normal
// scanning only has to be done on a few values to find an exact URL match.
const char kIndexEntriesRequest[] =
"CREATE INDEX entries_request_match_index "
"ON entries (cache_id, request_url_no_query_hash, "
"request_url_query_hash)";
const char kTableRequestHeaders[] =
"CREATE TABLE request_headers ("
"name TEXT NOT NULL, "
"value TEXT NOT NULL, "
"entry_id INTEGER NOT NULL REFERENCES entries(id) ON DELETE CASCADE"
")";
const char kTableResponseHeaders[] =
"CREATE TABLE response_headers ("
"name TEXT NOT NULL, "
"value TEXT NOT NULL, "
"entry_id INTEGER NOT NULL REFERENCES entries(id) ON DELETE CASCADE"
")";
// We need an index on response_headers, but not on request_headers,
// because we quickly need to determine if a VARY header is present.
const char kIndexResponseHeadersName[] =
"CREATE INDEX response_headers_name_index "
"ON response_headers (name)";
const char kTableResponseUrlList[] =
"CREATE TABLE response_url_list ("
"url TEXT NOT NULL, "
"entry_id INTEGER NOT NULL REFERENCES entries(id) ON DELETE CASCADE"
")";
// NOTE: key allows NULL below since that is how "" is represented
// in a BLOB column. We use BLOB to avoid encoding issues
// with storing DOMStrings.
const char kTableStorage[] =
"CREATE TABLE storage ("
"namespace INTEGER NOT NULL, "
"key BLOB NULL, "
"cache_id INTEGER NOT NULL REFERENCES caches(id), "
"PRIMARY KEY(namespace, key) "
")";
const char kTableUsageInfo[] =
"CREATE TABLE usage_info ("
"id INTEGER NOT NULL PRIMARY KEY, "
"total_disk_usage INTEGER NOT NULL "
")";
const char kTriggerEntriesInsert[] =
"CREATE TRIGGER entries_insert_trigger "
"AFTER INSERT ON entries "
"FOR EACH ROW "
"BEGIN "
"UPDATE usage_info SET total_disk_usage = total_disk_usage + "
"ifnull(NEW.request_body_disk_size, 0) + "
"ifnull(NEW.response_body_disk_size, 0) "
"WHERE usage_info.id = 1; "
"END";
const char kTriggerEntriesUpdate[] =
"CREATE TRIGGER entries_update_trigger "
"AFTER UPDATE ON entries "
"FOR EACH ROW "
"BEGIN "
"UPDATE usage_info SET total_disk_usage = total_disk_usage - "
"ifnull(OLD.request_body_disk_size, 0) + "
"ifnull(NEW.request_body_disk_size, 0) - "
"ifnull(OLD.response_body_disk_size, 0) + "
"ifnull(NEW.response_body_disk_size, 0) "
"WHERE usage_info.id = 1; "
"END";
const char kTriggerEntriesDelete[] =
"CREATE TRIGGER entries_delete_trigger "
"AFTER DELETE ON entries "
"FOR EACH ROW "
"BEGIN "
"UPDATE usage_info SET total_disk_usage = total_disk_usage - "
"ifnull(OLD.request_body_disk_size, 0) - "
"ifnull(OLD.response_body_disk_size, 0) "
"WHERE usage_info.id = 1; "
"END";
// ---------
// End schema definition
// ---------
const uint32_t kMaxEntriesPerStatement = 255;
const uint32_t kPageSize = 4 * 1024;
// Grow the database in chunks to reduce fragmentation
const uint32_t kGrowthSize = 32 * 1024;
const uint32_t kGrowthPages = kGrowthSize / kPageSize;
static_assert(kGrowthSize % kPageSize == 0,
"Growth size must be multiple of page size");
// Only release free pages when we have more than this limit
const int32_t kMaxFreePages = kGrowthPages;
// Limit WAL journal to a reasonable size
const uint32_t kWalAutoCheckpointSize = 512 * 1024;
const uint32_t kWalAutoCheckpointPages = kWalAutoCheckpointSize / kPageSize;
static_assert(kWalAutoCheckpointSize % kPageSize == 0,
"WAL checkpoint size must be multiple of page size");
} // namespace
// If any of the static_asserts below fail, it means that you have changed
// the corresponding WebIDL enum in a way that may be incompatible with the
// existing data stored in the DOM Cache. You would need to update the Cache
// database schema accordingly and adjust the failing static_assert.
static_assert(int(HeadersGuardEnum::None) == 0 &&
int(HeadersGuardEnum::Request) == 1 &&
int(HeadersGuardEnum::Request_no_cors) == 2 &&
int(HeadersGuardEnum::Response) == 3 &&
int(HeadersGuardEnum::Immutable) == 4 &&
ContiguousEnumSize<HeadersGuardEnum>::value == 5,
"HeadersGuardEnum values are as expected");
static_assert(int(ReferrerPolicy::_empty) == 0 &&
int(ReferrerPolicy::No_referrer) == 1 &&
int(ReferrerPolicy::No_referrer_when_downgrade) == 2 &&
int(ReferrerPolicy::Origin) == 3 &&
int(ReferrerPolicy::Origin_when_cross_origin) == 4 &&
int(ReferrerPolicy::Unsafe_url) == 5 &&
int(ReferrerPolicy::Same_origin) == 6 &&
int(ReferrerPolicy::Strict_origin) == 7 &&
int(ReferrerPolicy::Strict_origin_when_cross_origin) == 8 &&
ContiguousEnumSize<ReferrerPolicy>::value == 9,
"ReferrerPolicy values are as expected");
static_assert(int(RequestMode::Same_origin) == 0 &&
int(RequestMode::No_cors) == 1 &&
int(RequestMode::Cors) == 2 &&
int(RequestMode::Navigate) == 3 &&
ContiguousEnumSize<RequestMode>::value == 4,
"RequestMode values are as expected");
static_assert(int(RequestCredentials::Omit) == 0 &&
int(RequestCredentials::Same_origin) == 1 &&
int(RequestCredentials::Include) == 2 &&
ContiguousEnumSize<RequestCredentials>::value == 3,
"RequestCredentials values are as expected");
static_assert(int(RequestCache::Default) == 0 &&
int(RequestCache::No_store) == 1 &&
int(RequestCache::Reload) == 2 &&
int(RequestCache::No_cache) == 3 &&
int(RequestCache::Force_cache) == 4 &&
int(RequestCache::Only_if_cached) == 5 &&
ContiguousEnumSize<RequestCache>::value == 6,
"RequestCache values are as expected");
static_assert(int(RequestRedirect::Follow) == 0 &&
int(RequestRedirect::Error) == 1 &&
int(RequestRedirect::Manual) == 2 &&
ContiguousEnumSize<RequestRedirect>::value == 3,
"RequestRedirect values are as expected");
static_assert(int(ResponseType::Basic) == 0 && int(ResponseType::Cors) == 1 &&
int(ResponseType::Default) == 2 &&
int(ResponseType::Error) == 3 &&
int(ResponseType::Opaque) == 4 &&
int(ResponseType::Opaqueredirect) == 5 &&
ContiguousEnumSize<ResponseType>::value == 6,
"ResponseType values are as expected");
// If the static_asserts below fails, it means that you have changed the
// Namespace enum in a way that may be incompatible with the existing data
// stored in the DOM Cache. You would need to update the Cache database schema
// accordingly and adjust the failing static_assert.
static_assert(DEFAULT_NAMESPACE == 0 && CHROME_ONLY_NAMESPACE == 1 &&
NUMBER_OF_NAMESPACES == 2,
"Namespace values are as expected");
// If the static_asserts below fails, it means that you have changed the
// nsContentPolicy enum in a way that may be incompatible with the existing data
// stored in the DOM Cache. You would need to update the Cache database schema
// accordingly and adjust the failing static_assert.
static_assert(
nsIContentPolicy::TYPE_INVALID == 0 && nsIContentPolicy::TYPE_OTHER == 1 &&
nsIContentPolicy::TYPE_SCRIPT == 2 &&
nsIContentPolicy::TYPE_IMAGE == 3 &&
nsIContentPolicy::TYPE_STYLESHEET == 4 &&
nsIContentPolicy::TYPE_OBJECT == 5 &&
nsIContentPolicy::TYPE_DOCUMENT == 6 &&
nsIContentPolicy::TYPE_SUBDOCUMENT == 7 &&
nsIContentPolicy::TYPE_PING == 10 &&
nsIContentPolicy::TYPE_XMLHTTPREQUEST == 11 &&
nsIContentPolicy::TYPE_OBJECT_SUBREQUEST == 12 &&
nsIContentPolicy::TYPE_DTD == 13 && nsIContentPolicy::TYPE_FONT == 14 &&
nsIContentPolicy::TYPE_MEDIA == 15 &&
nsIContentPolicy::TYPE_WEBSOCKET == 16 &&
nsIContentPolicy::TYPE_CSP_REPORT == 17 &&
nsIContentPolicy::TYPE_XSLT == 18 &&
nsIContentPolicy::TYPE_BEACON == 19 &&
nsIContentPolicy::TYPE_FETCH == 20 &&
nsIContentPolicy::TYPE_IMAGESET == 21 &&
nsIContentPolicy::TYPE_WEB_MANIFEST == 22 &&
nsIContentPolicy::TYPE_INTERNAL_SCRIPT == 23 &&
nsIContentPolicy::TYPE_INTERNAL_WORKER == 24 &&
nsIContentPolicy::TYPE_INTERNAL_SHARED_WORKER == 25 &&
nsIContentPolicy::TYPE_INTERNAL_EMBED == 26 &&
nsIContentPolicy::TYPE_INTERNAL_OBJECT == 27 &&
nsIContentPolicy::TYPE_INTERNAL_FRAME == 28 &&
nsIContentPolicy::TYPE_INTERNAL_IFRAME == 29 &&
nsIContentPolicy::TYPE_INTERNAL_AUDIO == 30 &&
nsIContentPolicy::TYPE_INTERNAL_VIDEO == 31 &&
nsIContentPolicy::TYPE_INTERNAL_TRACK == 32 &&
nsIContentPolicy::TYPE_INTERNAL_XMLHTTPREQUEST_ASYNC == 33 &&
nsIContentPolicy::TYPE_INTERNAL_EVENTSOURCE == 34 &&
nsIContentPolicy::TYPE_INTERNAL_SERVICE_WORKER == 35 &&
nsIContentPolicy::TYPE_INTERNAL_SCRIPT_PRELOAD == 36 &&
nsIContentPolicy::TYPE_INTERNAL_IMAGE == 37 &&
nsIContentPolicy::TYPE_INTERNAL_IMAGE_PRELOAD == 38 &&
nsIContentPolicy::TYPE_INTERNAL_STYLESHEET == 39 &&
nsIContentPolicy::TYPE_INTERNAL_STYLESHEET_PRELOAD == 40 &&
nsIContentPolicy::TYPE_INTERNAL_IMAGE_FAVICON == 41 &&
nsIContentPolicy::TYPE_INTERNAL_WORKER_IMPORT_SCRIPTS == 42 &&
nsIContentPolicy::TYPE_SAVEAS_DOWNLOAD == 43 &&
nsIContentPolicy::TYPE_SPECULATIVE == 44 &&
nsIContentPolicy::TYPE_INTERNAL_MODULE == 45 &&
nsIContentPolicy::TYPE_INTERNAL_MODULE_PRELOAD == 46 &&
nsIContentPolicy::TYPE_INTERNAL_DTD == 47 &&
nsIContentPolicy::TYPE_INTERNAL_FORCE_ALLOWED_DTD == 48 &&
nsIContentPolicy::TYPE_INTERNAL_AUDIOWORKLET == 49 &&
nsIContentPolicy::TYPE_INTERNAL_PAINTWORKLET == 50 &&
nsIContentPolicy::TYPE_INTERNAL_FONT_PRELOAD == 51 &&
nsIContentPolicy::TYPE_INTERNAL_CHROMEUTILS_COMPILED_SCRIPT == 52 &&
nsIContentPolicy::TYPE_INTERNAL_FRAME_MESSAGEMANAGER_SCRIPT == 53 &&
nsIContentPolicy::TYPE_INTERNAL_FETCH_PRELOAD == 54 &&
nsIContentPolicy::TYPE_UA_FONT == 55 &&
nsIContentPolicy::TYPE_WEB_IDENTITY == 57 &&
nsIContentPolicy::TYPE_INTERNAL_WORKER_STATIC_MODULE == 58 &&
nsIContentPolicy::TYPE_WEB_TRANSPORT == 59 &&
nsIContentPolicy::TYPE_INTERNAL_XMLHTTPREQUEST_SYNC == 60 &&
nsIContentPolicy::TYPE_INTERNAL_EXTERNAL_RESOURCE == 61 &&
nsIContentPolicy::TYPE_END == 62,
"nsContentPolicyType values are as expected");
namespace {
using EntryId = int32_t;
struct IdCount {
explicit IdCount(int32_t aId) : mId(aId), mCount(1) {}
int32_t mId;
int32_t mCount;
};
using EntryIds = AutoTArray<EntryId, 256>;
static Result<EntryIds, nsresult> QueryAll(mozIStorageConnection& aConn,
CacheId aCacheId);
static Result<EntryIds, nsresult> QueryCache(mozIStorageConnection& aConn,
CacheId aCacheId,
const CacheRequest& aRequest,
const CacheQueryParams& aParams,
uint32_t aMaxResults = UINT32_MAX);
static Result<bool, nsresult> MatchByVaryHeader(mozIStorageConnection& aConn,
const CacheRequest& aRequest,
EntryId entryId);
// Returns a success tuple containing the deleted body ids, deleted security ids
// and deleted padding size.
static Result<std::tuple<nsTArray<nsID>, AutoTArray<IdCount, 16>, int64_t>,
nsresult>
DeleteEntries(mozIStorageConnection& aConn,
const nsTArray<EntryId>& aEntryIdList);
static Result<std::tuple<nsTArray<nsID>, AutoTArray<IdCount, 16>, int64_t>,
nsresult>
DeleteAllCacheEntries(mozIStorageConnection& aConn, CacheId& aCacheId);
static Result<int32_t, nsresult> InsertSecurityInfo(
mozIStorageConnection& aConn, nsICryptoHash& aCrypto,
nsITransportSecurityInfo* aSecurityInfo);
static nsresult DeleteSecurityInfo(mozIStorageConnection& aConn, int32_t aId,
int32_t aCount);
static nsresult DeleteSecurityInfoList(
mozIStorageConnection& aConn,
const nsTArray<IdCount>& aDeletedStorageIdList);
static nsresult InsertEntry(mozIStorageConnection& aConn, CacheId aCacheId,
const CacheRequest& aRequest,
const nsID* aRequestBodyId,
const CacheResponse& aResponse,
const nsID* aResponseBodyId);
static Result<SavedResponse, nsresult> ReadResponse(
mozIStorageConnection& aConn, EntryId aEntryId);
static Result<SavedRequest, nsresult> ReadRequest(mozIStorageConnection& aConn,
EntryId aEntryId);
static void AppendListParamsToQuery(nsACString& aQuery, size_t aLen);
static nsresult BindListParamsToQuery(mozIStorageStatement& aState,
const Span<const EntryId>& aEntryIdList);
static nsresult BindId(mozIStorageStatement& aState, const nsACString& aName,
const nsID* aId);
static Result<nsID, nsresult> ExtractId(mozIStorageStatement& aState,
uint32_t aPos);
static Result<NotNull<nsCOMPtr<mozIStorageStatement>>, nsresult>
CreateAndBindKeyStatement(mozIStorageConnection& aConn,
const char* aQueryFormat, const nsAString& aKey);
static Result<nsAutoCString, nsresult> HashCString(nsICryptoHash& aCrypto,
const nsACString& aIn);
Result<int32_t, nsresult> GetEffectiveSchemaVersion(
mozIStorageConnection& aConn);
nsresult Validate(mozIStorageConnection& aConn);
nsresult Migrate(nsIFile& aDBDir, mozIStorageConnection& aConn);
} // namespace
class MOZ_RAII AutoDisableForeignKeyChecking {
public:
explicit AutoDisableForeignKeyChecking(mozIStorageConnection* aConn)
: mConn(aConn), mForeignKeyCheckingDisabled(false) {
QM_TRY_INSPECT(const auto& state,
quota::CreateAndExecuteSingleStepStatement(
*mConn, "PRAGMA foreign_keys;"_ns),
QM_VOID);
QM_TRY_INSPECT(const int32_t& mode,
MOZ_TO_RESULT_INVOKE_MEMBER(*state, GetInt32, 0), QM_VOID);
if (mode) {
QM_WARNONLY_TRY(MOZ_TO_RESULT(mConn->ExecuteSimpleSQL(
"PRAGMA foreign_keys = OFF;"_ns))
.andThen([this](const auto) -> Result<Ok, nsresult> {
mForeignKeyCheckingDisabled = true;
return Ok{};
}));
}
}
~AutoDisableForeignKeyChecking() {
if (mForeignKeyCheckingDisabled) {
QM_WARNONLY_TRY(QM_TO_RESULT(
mConn->ExecuteSimpleSQL("PRAGMA foreign_keys = ON;"_ns)));
}
}
private:
nsCOMPtr<mozIStorageConnection> mConn;
bool mForeignKeyCheckingDisabled;
};
nsresult CreateOrMigrateSchema(nsIFile& aDBDir, mozIStorageConnection& aConn) {
MOZ_ASSERT(!NS_IsMainThread());
QM_TRY_UNWRAP(int32_t schemaVersion, GetEffectiveSchemaVersion(aConn));
if (schemaVersion == kLatestSchemaVersion) {
// We already have the correct schema version. Validate it matches
// our expected schema and then proceed.
QM_TRY(MOZ_TO_RESULT(Validate(aConn)));
return NS_OK;
}
// Turn off checking foreign keys before starting a transaction, and restore
// it once we're done.
AutoDisableForeignKeyChecking restoreForeignKeyChecking(&aConn);
mozStorageTransaction trans(&aConn, false,
mozIStorageConnection::TRANSACTION_IMMEDIATE);
QM_TRY(MOZ_TO_RESULT(trans.Start()));
const bool migrating = schemaVersion != 0;
if (migrating) {
// A schema exists, but its not the current version. Attempt to
// migrate it to our new schema.
QM_TRY(MOZ_TO_RESULT(Migrate(aDBDir, aConn)));
} else {
// There is no schema installed. Create the database from scratch.
QM_TRY(
MOZ_TO_RESULT(aConn.ExecuteSimpleSQL(nsLiteralCString(kTableCaches))));
QM_TRY(MOZ_TO_RESULT(
aConn.ExecuteSimpleSQL(nsLiteralCString(kTableSecurityInfo))));
QM_TRY(MOZ_TO_RESULT(
aConn.ExecuteSimpleSQL(nsLiteralCString(kIndexSecurityInfoHash))));
QM_TRY(
MOZ_TO_RESULT(aConn.ExecuteSimpleSQL(nsLiteralCString(kTableEntries))));
QM_TRY(MOZ_TO_RESULT(
aConn.ExecuteSimpleSQL(nsLiteralCString(kIndexEntriesRequest))));
QM_TRY(MOZ_TO_RESULT(
aConn.ExecuteSimpleSQL(nsLiteralCString(kTableRequestHeaders))));
QM_TRY(MOZ_TO_RESULT(
aConn.ExecuteSimpleSQL(nsLiteralCString(kTableResponseHeaders))));
QM_TRY(MOZ_TO_RESULT(
aConn.ExecuteSimpleSQL(nsLiteralCString(kIndexResponseHeadersName))));
QM_TRY(MOZ_TO_RESULT(
aConn.ExecuteSimpleSQL(nsLiteralCString(kTableResponseUrlList))));
QM_TRY(
MOZ_TO_RESULT(aConn.ExecuteSimpleSQL(nsLiteralCString(kTableStorage))));
QM_TRY(MOZ_TO_RESULT(
aConn.ExecuteSimpleSQL(nsLiteralCString(kTableUsageInfo))));
QM_TRY(MOZ_TO_RESULT(aConn.ExecuteSimpleSQL(
nsLiteralCString("INSERT INTO usage_info VALUES(1, 0);"))));
QM_TRY(MOZ_TO_RESULT(
aConn.ExecuteSimpleSQL(nsLiteralCString(kTriggerEntriesInsert))));
QM_TRY(MOZ_TO_RESULT(
aConn.ExecuteSimpleSQL(nsLiteralCString(kTriggerEntriesUpdate))));
QM_TRY(MOZ_TO_RESULT(
aConn.ExecuteSimpleSQL(nsLiteralCString(kTriggerEntriesDelete))));
QM_TRY(MOZ_TO_RESULT(aConn.SetSchemaVersion(kLatestSchemaVersion)));
QM_TRY_UNWRAP(schemaVersion, GetEffectiveSchemaVersion(aConn));
}
QM_TRY(MOZ_TO_RESULT(Validate(aConn)));
QM_TRY(MOZ_TO_RESULT(trans.Commit()));
if (migrating) {
// Migrations happen infrequently and reflect a chance in DB structure.
// This is a good time to rebuild the database. It also helps catch
// if a new migration is incorrect by fast failing on the corruption.
// Unfortunately, this must be performed outside of the transaction.
QM_TRY(MOZ_TO_RESULT(aConn.ExecuteSimpleSQL("VACUUM"_ns)));
}
return NS_OK;
}
nsresult InitializeConnection(mozIStorageConnection& aConn) {
MOZ_ASSERT(!NS_IsMainThread());
// This function needs to perform per-connection initialization tasks that
// need to happen regardless of the schema.
// Note, the default encoding of UTF-8 is preferred. mozStorage does all
// the work necessary to convert UTF-16 nsString values for us. We don't
// need ordering and the binary equality operations are correct. So, do
// NOT set PRAGMA encoding to UTF-16.
QM_TRY(MOZ_TO_RESULT(aConn.ExecuteSimpleSQL(nsPrintfCString(
// Use a smaller page size to improve perf/footprint; default is too large
"PRAGMA page_size = %u; "
// Enable auto_vacuum; this must happen after page_size and before WAL
"PRAGMA auto_vacuum = INCREMENTAL; "
"PRAGMA foreign_keys = ON; ",
kPageSize))));
// Limit fragmentation by growing the database by many pages at once.
QM_TRY(QM_OR_ELSE_WARN_IF(
// Expression.
MOZ_TO_RESULT(aConn.SetGrowthIncrement(kGrowthSize, ""_ns)),
// Predicate.
IsSpecificError<NS_ERROR_FILE_TOO_BIG>,
// Fallback.
ErrToDefaultOk<>));
// Enable WAL journaling. This must be performed in a separate transaction
// after changing the page_size and enabling auto_vacuum.
// Note there is a default journal_size_limit set by mozStorage.
QM_TRY(MOZ_TO_RESULT(aConn.ExecuteSimpleSQL(nsPrintfCString(
// WAL journal can grow to given number of *pages*
"PRAGMA wal_autocheckpoint = %u; "
// WAL must be enabled at the end to allow page size to be changed, etc.
"PRAGMA journal_mode = WAL; ",
kWalAutoCheckpointPages))));
// Verify that we successfully set the vacuum mode to incremental. It
// is very easy to put the database in a state where the auto_vacuum
// pragma above fails silently.
#ifdef DEBUG
{
QM_TRY_INSPECT(const auto& state,
quota::CreateAndExecuteSingleStepStatement(
aConn, "PRAGMA auto_vacuum;"_ns));
QM_TRY_INSPECT(const int32_t& mode,
MOZ_TO_RESULT_INVOKE_MEMBER(*state, GetInt32, 0));
// integer value 2 is incremental mode
QM_TRY(OkIf(mode == 2), NS_ERROR_UNEXPECTED);
}
#endif
return NS_OK;
}
Result<CacheId, nsresult> CreateCacheId(mozIStorageConnection& aConn) {
MOZ_ASSERT(!NS_IsMainThread());
QM_TRY(MOZ_TO_RESULT(
aConn.ExecuteSimpleSQL("INSERT INTO caches DEFAULT VALUES;"_ns)));
QM_TRY_INSPECT(const auto& state,
quota::CreateAndExecuteSingleStepStatement<
quota::SingleStepResult::ReturnNullIfNoResult>(
aConn, "SELECT last_insert_rowid()"_ns));
QM_TRY(OkIf(state), Err(NS_ERROR_UNEXPECTED));
QM_TRY_INSPECT(const CacheId& id,
MOZ_TO_RESULT_INVOKE_MEMBER(state, GetInt64, 0));
return id;
}
Result<DeletionInfo, nsresult> DeleteCacheId(mozIStorageConnection& aConn,
CacheId aCacheId) {
MOZ_ASSERT(!NS_IsMainThread());
// XXX only deletedBodyIdList needs to be non-const
QM_TRY_UNWRAP(
(auto [deletedBodyIdList, deletedSecurityIdList, deletedPaddingSize]),
DeleteAllCacheEntries(aConn, aCacheId));
QM_TRY(MOZ_TO_RESULT(DeleteSecurityInfoList(aConn, deletedSecurityIdList)));
// Delete the remainder of the cache using cascade semantics.
QM_TRY_INSPECT(const auto& state,
MOZ_TO_RESULT_INVOKE_MEMBER_TYPED(
nsCOMPtr<mozIStorageStatement>, aConn, CreateStatement,
"DELETE FROM caches WHERE id=:id;"_ns));
QM_TRY(MOZ_TO_RESULT(state->BindInt64ByName("id"_ns, aCacheId)));
QM_TRY(MOZ_TO_RESULT(state->Execute()));
return DeletionInfo{std::move(deletedBodyIdList), deletedPaddingSize};
}
Result<AutoTArray<CacheId, 8>, nsresult> FindOrphanedCacheIds(
mozIStorageConnection& aConn) {
QM_TRY_INSPECT(const auto& state,
MOZ_TO_RESULT_INVOKE_MEMBER_TYPED(
nsCOMPtr<mozIStorageStatement>, aConn, CreateStatement,
"SELECT id FROM caches "
"WHERE id NOT IN (SELECT cache_id from storage);"_ns));
QM_TRY_RETURN(
(quota::CollectElementsWhileHasResultTyped<AutoTArray<CacheId, 8>>(
*state, [](auto& stmt) {
QM_TRY_RETURN(MOZ_TO_RESULT_INVOKE_MEMBER(stmt, GetInt64, 0));
})));
}
Result<int64_t, nsresult> FindOverallPaddingSize(mozIStorageConnection& aConn) {
QM_TRY_INSPECT(const auto& state,
MOZ_TO_RESULT_INVOKE_MEMBER_TYPED(
nsCOMPtr<mozIStorageStatement>, aConn, CreateStatement,
"SELECT response_padding_size FROM entries "
"WHERE response_padding_size IS NOT NULL;"_ns));
int64_t overallPaddingSize = 0;
QM_TRY(quota::CollectWhileHasResult(
*state, [&overallPaddingSize](auto& stmt) -> Result<Ok, nsresult> {
QM_TRY_INSPECT(const int64_t& padding_size,
MOZ_TO_RESULT_INVOKE_MEMBER(stmt, GetInt64, 0));
MOZ_DIAGNOSTIC_ASSERT(padding_size >= 0);
MOZ_DIAGNOSTIC_ASSERT(INT64_MAX - padding_size >= overallPaddingSize);
overallPaddingSize += padding_size;
return Ok{};
}));
return overallPaddingSize;
}
Result<int64_t, nsresult> GetTotalDiskUsage(mozIStorageConnection& aConn) {
QM_TRY_INSPECT(
const auto& state,
quota::CreateAndExecuteSingleStepStatement(
aConn, "SELECT total_disk_usage FROM usage_info WHERE id = 1;"_ns));
QM_TRY_RETURN(MOZ_TO_RESULT_INVOKE_MEMBER(*state, GetInt64, 0));
}
Result<nsTArray<nsID>, nsresult> GetKnownBodyIds(mozIStorageConnection& aConn) {
MOZ_ASSERT(!NS_IsMainThread());
QM_TRY_INSPECT(
const auto& state,
MOZ_TO_RESULT_INVOKE_MEMBER_TYPED(
nsCOMPtr<mozIStorageStatement>, aConn, CreateStatement,
"SELECT request_body_id, response_body_id FROM entries;"_ns));
AutoTArray<nsID, 64> idList;
QM_TRY(quota::CollectWhileHasResult(
*state, [&idList](auto& stmt) -> Result<Ok, nsresult> {
// extract 0 to 2 nsID structs per row
for (uint32_t i = 0; i < 2; ++i) {
QM_TRY_INSPECT(const bool& isNull,
MOZ_TO_RESULT_INVOKE_MEMBER(stmt, GetIsNull, i));
if (!isNull) {
QM_TRY_INSPECT(const auto& id, ExtractId(stmt, i));
idList.AppendElement(id);
}
}
return Ok{};
}));
return std::move(idList);
}
Result<Maybe<SavedResponse>, nsresult> CacheMatch(
mozIStorageConnection& aConn, CacheId aCacheId,
const CacheRequest& aRequest, const CacheQueryParams& aParams) {
MOZ_ASSERT(!NS_IsMainThread());
QM_TRY_INSPECT(const auto& matches,
QueryCache(aConn, aCacheId, aRequest, aParams, 1));
if (matches.IsEmpty()) {
return Maybe<SavedResponse>();
}
QM_TRY_UNWRAP(auto response, ReadResponse(aConn, matches[0]));
response.mCacheId = aCacheId;
return Some(std::move(response));
}
Result<nsTArray<SavedResponse>, nsresult> CacheMatchAll(
mozIStorageConnection& aConn, CacheId aCacheId,
const Maybe<CacheRequest>& aMaybeRequest, const CacheQueryParams& aParams) {
MOZ_ASSERT(!NS_IsMainThread());
QM_TRY_INSPECT(
const auto& matches, ([&aConn, aCacheId, &aMaybeRequest, &aParams] {
if (aMaybeRequest.isNothing()) {
QM_TRY_RETURN(QueryAll(aConn, aCacheId));
}
QM_TRY_RETURN(
QueryCache(aConn, aCacheId, aMaybeRequest.ref(), aParams));
}()));
QM_TRY_RETURN(TransformIntoNewArrayAbortOnErr(
matches,
[&aConn, aCacheId](const auto match) -> Result<SavedResponse, nsresult> {
QM_TRY_UNWRAP(auto savedResponse, ReadResponse(aConn, match));
savedResponse.mCacheId = aCacheId;
return savedResponse;
},
fallible));
}
Result<DeletionInfo, nsresult> CachePut(mozIStorageConnection& aConn,
CacheId aCacheId,
const CacheRequest& aRequest,
const nsID* aRequestBodyId,
const CacheResponse& aResponse,
const nsID* aResponseBodyId) {
MOZ_ASSERT(!NS_IsMainThread());
QM_TRY_INSPECT(
const auto& matches,
QueryCache(aConn, aCacheId, aRequest,
CacheQueryParams(false, false, false, false, u""_ns)));
// XXX only deletedBodyIdList needs to be non-const
QM_TRY_UNWRAP(
(auto [deletedBodyIdList, deletedSecurityIdList, deletedPaddingSize]),
DeleteEntries(aConn, matches));
QM_TRY(MOZ_TO_RESULT(InsertEntry(aConn, aCacheId, aRequest, aRequestBodyId,
aResponse, aResponseBodyId)));
// Delete the security values after doing the insert to avoid churning
// the security table when its not necessary.
QM_TRY(MOZ_TO_RESULT(DeleteSecurityInfoList(aConn, deletedSecurityIdList)));
return DeletionInfo{std::move(deletedBodyIdList), deletedPaddingSize};
}
Result<Maybe<DeletionInfo>, nsresult> CacheDelete(
mozIStorageConnection& aConn, CacheId aCacheId,
const CacheRequest& aRequest, const CacheQueryParams& aParams) {
MOZ_ASSERT(!NS_IsMainThread());
QM_TRY_INSPECT(const auto& matches,
QueryCache(aConn, aCacheId, aRequest, aParams));
if (matches.IsEmpty()) {
return Maybe<DeletionInfo>();
}
// XXX only deletedBodyIdList needs to be non-const
QM_TRY_UNWRAP(
(auto [deletedBodyIdList, deletedSecurityIdList, deletedPaddingSize]),
DeleteEntries(aConn, matches));
QM_TRY(MOZ_TO_RESULT(DeleteSecurityInfoList(aConn, deletedSecurityIdList)));
return Some(DeletionInfo{std::move(deletedBodyIdList), deletedPaddingSize});
}
Result<nsTArray<SavedRequest>, nsresult> CacheKeys(
mozIStorageConnection& aConn, CacheId aCacheId,
const Maybe<CacheRequest>& aMaybeRequest, const CacheQueryParams& aParams) {
MOZ_ASSERT(!NS_IsMainThread());
QM_TRY_INSPECT(
const auto& matches, ([&aConn, aCacheId, &aMaybeRequest, &aParams] {
if (aMaybeRequest.isNothing()) {
QM_TRY_RETURN(QueryAll(aConn, aCacheId));
}
QM_TRY_RETURN(
QueryCache(aConn, aCacheId, aMaybeRequest.ref(), aParams));
}()));
QM_TRY_RETURN(TransformIntoNewArrayAbortOnErr(
matches,
[&aConn, aCacheId](const auto match) -> Result<SavedRequest, nsresult> {
QM_TRY_UNWRAP(auto savedRequest, ReadRequest(aConn, match));
savedRequest.mCacheId = aCacheId;
return savedRequest;
},
fallible));
}
Result<Maybe<SavedResponse>, nsresult> StorageMatch(
mozIStorageConnection& aConn, Namespace aNamespace,
const CacheRequest& aRequest, const CacheQueryParams& aParams) {