From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Scott Hess <shess@chromium.org>
Date: Thu, 2 Mar 2017 15:23:09 -0800
Subject: [PATCH 07/10] Allow auto-vacuum to work with chunks.

SQLITE_FCNTL_CHUNK_SIZE can advise the VFS to resize files in quantum
amounts, to reduce fragmentation from tiny appends.  This change allows
a new PRAGMA auto_vacuum_slack_pages to provide auto_vacuum with a hint
to only rearrange pages when an entire quantum can be released.

When rebasing this patch, first ignore the conflicts in src/pragma.h,
and fix all the other conflicts. Then run the commands below (in
third_party/sqlite) to re-generate src/pragma.h.
tclsh src/tool/mkpragmatab.tcl
find src/ -type f -iname "*.h" -exec \
    $GNU_SED --in-place 's/[[:space:]]\+$//' {} \+

BUG=698010
---
 third_party/sqlite/src/src/btree.c          | 56 +++++++++++-
 third_party/sqlite/src/src/btree.h          |  2 +
 third_party/sqlite/src/src/btreeInt.h       |  1 +
 third_party/sqlite/src/src/pragma.c         | 21 +++++
 third_party/sqlite/src/src/pragma.h         | 96 +++++++++++----------
 third_party/sqlite/src/tool/mkpragmatab.tcl |  4 +
 6 files changed, 134 insertions(+), 46 deletions(-)

diff --git a/third_party/sqlite/src/src/btree.c b/third_party/sqlite/src/src/btree.c
index b125859bf570..3fa8f88f1465 100644
--- a/third_party/sqlite/src/src/btree.c
+++ b/third_party/sqlite/src/src/btree.c
@@ -2980,6 +2980,46 @@ static void setDefaultSyncFlag(BtShared *pBt, u8 safety_level){
 static int newDatabase(BtShared*);


+/*
+** Change the 'auto-vacuum-slack-pages' property of the database. If auto vacuum
+** is enabled, this is the number of chunks of slack to allow before
+** automatically running an incremental vacuum.
+*/
+int sqlite3BtreeSetAutoVacuumSlackPages(Btree *p, int autoVacuumSlack){
+#ifdef SQLITE_OMIT_AUTOVACUUM
+  return SQLITE_READONLY;
+#else
+  BtShared *pBt = p->pBt;
+  int rc = SQLITE_OK;
+  u8 avs = (u8)autoVacuumSlack;
+  if( autoVacuumSlack>avs ){
+    avs = 0xFF;
+  }
+
+  sqlite3BtreeEnter(p);
+  pBt->autoVacuumSlack = avs;
+  sqlite3BtreeLeave(p);
+  return rc;
+#endif
+}
+
+/*
+** Return the value of the 'auto-vacuum-slack-pages' property.
+*/
+int sqlite3BtreeGetAutoVacuumSlackPages(Btree *p){
+#ifdef SQLITE_OMIT_AUTOVACUUM
+  return 0;
+#else
+  int rc = 0;
+  sqlite3BtreeEnter(p);
+  if( p->pBt->autoVacuum!=0 ){
+    rc = p->pBt->autoVacuumSlack;
+  }
+  sqlite3BtreeLeave(p);
+  return rc;
+#endif
+}
+
 /*
 ** Get a reference to pPage1 of the database file.  This will
 ** also acquire a readlock on that file.
@@ -3828,13 +3868,27 @@ int sqlite3BtreeIncrVacuum(Btree *p){
 */
 static int autoVacuumCommit(BtShared *pBt){
   int rc = SQLITE_OK;
+  int bShouldVacuum = pBt->autoVacuum && !pBt->incrVacuum;
   Pager *pPager = pBt->pPager;
   VVA_ONLY( int nRef = sqlite3PagerRefcount(pPager); )

   assert( sqlite3_mutex_held(pBt->mutex) );
   invalidateAllOverflowCache(pBt);
   assert(pBt->autoVacuum);
-  if( !pBt->incrVacuum ){
+  if( bShouldVacuum && pBt->autoVacuumSlack ){
+    Pgno nOrig;        /* Database size before freeing */
+    Pgno nFree;        /* Number of pages on the freelist initially */
+
+    nOrig = btreePagecount(pBt);
+    nFree = get4byte(&pBt->pPage1->aData[36]);
+    bShouldVacuum =
+        (nOrig-nFree-1)/pBt->autoVacuumSlack < (nOrig-1)/pBt->autoVacuumSlack;
+    /* TODO: When integrating this test with the following code, contrive to
+    ** trim to the integral chunk boundary, rather than trimming the entire free
+    ** list.
+    */
+  }
+  if( bShouldVacuum ){
     Pgno nFin;         /* Number of pages in database after autovacuuming */
     Pgno nFree;        /* Number of pages on the freelist initially */
     Pgno iFree;        /* The next page to be freed */
diff --git a/third_party/sqlite/src/src/btree.h b/third_party/sqlite/src/src/btree.h
index e4bd09c7e727..83307a95ad3f 100644
--- a/third_party/sqlite/src/src/btree.h
+++ b/third_party/sqlite/src/src/btree.h
@@ -78,6 +78,8 @@ int sqlite3BtreeGetOptimalReserve(Btree*);
 int sqlite3BtreeGetReserveNoMutex(Btree *p);
 int sqlite3BtreeSetAutoVacuum(Btree *, int);
 int sqlite3BtreeGetAutoVacuum(Btree *);
+int sqlite3BtreeSetAutoVacuumSlackPages(Btree *, int);
+int sqlite3BtreeGetAutoVacuumSlackPages(Btree *);
 int sqlite3BtreeBeginTrans(Btree*,int,int*);
 int sqlite3BtreeCommitPhaseOne(Btree*, const char *zMaster);
 int sqlite3BtreeCommitPhaseTwo(Btree*, int);
diff --git a/third_party/sqlite/src/src/btreeInt.h b/third_party/sqlite/src/src/btreeInt.h
index 8fe8e280fe5f..88f3b437b8ea 100644
--- a/third_party/sqlite/src/src/btreeInt.h
+++ b/third_party/sqlite/src/src/btreeInt.h
@@ -412,6 +412,7 @@ struct BtShared {
   u8 openFlags;         /* Flags to sqlite3BtreeOpen() */
 #ifndef SQLITE_OMIT_AUTOVACUUM
   u8 autoVacuum;        /* True if auto-vacuum is enabled */
+  u8 autoVacuumSlack;   /* Optional pages of slack for auto-vacuum */
   u8 incrVacuum;        /* True if incr-vacuum is enabled */
   u8 bDoTruncate;       /* True to truncate db on commit */
 #endif
diff --git a/third_party/sqlite/src/src/pragma.c b/third_party/sqlite/src/src/pragma.c
index 6e026557d561..472fca7ee699 100644
--- a/third_party/sqlite/src/src/pragma.c
+++ b/third_party/sqlite/src/src/pragma.c
@@ -756,6 +756,27 @@ void sqlite3Pragma(
   }
 #endif

+  /*
+  **  PRAGMA [schema.]auto_vacuum_slack_pages(N)
+  **
+  ** Control chunk size of auto-vacuum.
+  */
+#ifndef SQLITE_OMIT_AUTOVACUUM
+  case PragTyp_AUTO_VACUUM_SLACK_PAGES: {
+    Btree *pBt = pDb->pBt;
+    assert( pBt!=0 );
+    if( !zRight ){
+      returnSingleInt(v, sqlite3BtreeGetAutoVacuumSlackPages(pBt));
+    }else{
+      int nPages = 8;
+      if( sqlite3GetInt32(zRight, &nPages) ){
+        sqlite3BtreeSetAutoVacuumSlackPages(pBt, nPages);
+      }
+    }
+    break;
+  }
+#endif
+
 #ifndef SQLITE_OMIT_PAGER_PRAGMAS
   /*
   **  PRAGMA [schema.]cache_size
diff --git a/third_party/sqlite/src/src/pragma.h b/third_party/sqlite/src/src/pragma.h
index 826516202e41..970092f1cd8d 100644
--- a/third_party/sqlite/src/src/pragma.h
+++ b/third_party/sqlite/src/src/pragma.h
@@ -7,51 +7,52 @@
 /* The various pragma types */
 #define PragTyp_HEADER_VALUE                   0
 #define PragTyp_AUTO_VACUUM                    1
-#define PragTyp_FLAG                           2
-#define PragTyp_BUSY_TIMEOUT                   3
-#define PragTyp_CACHE_SIZE                     4
-#define PragTyp_CACHE_SPILL                    5
-#define PragTyp_CASE_SENSITIVE_LIKE            6
-#define PragTyp_COLLATION_LIST                 7
-#define PragTyp_COMPILE_OPTIONS                8
-#define PragTyp_DATA_STORE_DIRECTORY           9
-#define PragTyp_DATABASE_LIST                 10
-#define PragTyp_DEFAULT_CACHE_SIZE            11
-#define PragTyp_ENCODING                      12
-#define PragTyp_FOREIGN_KEY_CHECK             13
-#define PragTyp_FOREIGN_KEY_LIST              14
-#define PragTyp_FUNCTION_LIST                 15
-#define PragTyp_INCREMENTAL_VACUUM            16
-#define PragTyp_INDEX_INFO                    17
-#define PragTyp_INDEX_LIST                    18
-#define PragTyp_INTEGRITY_CHECK               19
-#define PragTyp_JOURNAL_MODE                  20
-#define PragTyp_JOURNAL_SIZE_LIMIT            21
-#define PragTyp_LOCK_PROXY_FILE               22
-#define PragTyp_LOCKING_MODE                  23
-#define PragTyp_PAGE_COUNT                    24
-#define PragTyp_MMAP_SIZE                     25
-#define PragTyp_MODULE_LIST                   26
-#define PragTyp_OPTIMIZE                      27
-#define PragTyp_PAGE_SIZE                     28
-#define PragTyp_PRAGMA_LIST                   29
-#define PragTyp_SECURE_DELETE                 30
-#define PragTyp_SHRINK_MEMORY                 31
-#define PragTyp_SOFT_HEAP_LIMIT               32
-#define PragTyp_SYNCHRONOUS                   33
-#define PragTyp_TABLE_INFO                    34
-#define PragTyp_TEMP_STORE                    35
-#define PragTyp_TEMP_STORE_DIRECTORY          36
-#define PragTyp_THREADS                       37
-#define PragTyp_WAL_AUTOCHECKPOINT            38
-#define PragTyp_WAL_CHECKPOINT                39
-#define PragTyp_ACTIVATE_EXTENSIONS           40
-#define PragTyp_HEXKEY                        41
-#define PragTyp_KEY                           42
-#define PragTyp_REKEY                         43
-#define PragTyp_LOCK_STATUS                   44
-#define PragTyp_PARSER_TRACE                  45
-#define PragTyp_STATS                         46
+#define PragTyp_AUTO_VACUUM_SLACK_PAGES        2
+#define PragTyp_FLAG                           3
+#define PragTyp_BUSY_TIMEOUT                   4
+#define PragTyp_CACHE_SIZE                     5
+#define PragTyp_CACHE_SPILL                    6
+#define PragTyp_CASE_SENSITIVE_LIKE            7
+#define PragTyp_COLLATION_LIST                 8
+#define PragTyp_COMPILE_OPTIONS                9
+#define PragTyp_DATA_STORE_DIRECTORY          10
+#define PragTyp_DATABASE_LIST                 11
+#define PragTyp_DEFAULT_CACHE_SIZE            12
+#define PragTyp_ENCODING                      13
+#define PragTyp_FOREIGN_KEY_CHECK             14
+#define PragTyp_FOREIGN_KEY_LIST              15
+#define PragTyp_FUNCTION_LIST                 16
+#define PragTyp_INCREMENTAL_VACUUM            17
+#define PragTyp_INDEX_INFO                    18
+#define PragTyp_INDEX_LIST                    19
+#define PragTyp_INTEGRITY_CHECK               20
+#define PragTyp_JOURNAL_MODE                  21
+#define PragTyp_JOURNAL_SIZE_LIMIT            22
+#define PragTyp_LOCK_PROXY_FILE               23
+#define PragTyp_LOCKING_MODE                  24
+#define PragTyp_PAGE_COUNT                    25
+#define PragTyp_MMAP_SIZE                     26
+#define PragTyp_MODULE_LIST                   27
+#define PragTyp_OPTIMIZE                      28
+#define PragTyp_PAGE_SIZE                     29
+#define PragTyp_PRAGMA_LIST                   30
+#define PragTyp_SECURE_DELETE                 31
+#define PragTyp_SHRINK_MEMORY                 32
+#define PragTyp_SOFT_HEAP_LIMIT               33
+#define PragTyp_SYNCHRONOUS                   34
+#define PragTyp_TABLE_INFO                    35
+#define PragTyp_TEMP_STORE                    36
+#define PragTyp_TEMP_STORE_DIRECTORY          37
+#define PragTyp_THREADS                       38
+#define PragTyp_WAL_AUTOCHECKPOINT            39
+#define PragTyp_WAL_CHECKPOINT                40
+#define PragTyp_ACTIVATE_EXTENSIONS           41
+#define PragTyp_HEXKEY                        42
+#define PragTyp_KEY                           43
+#define PragTyp_REKEY                         44
+#define PragTyp_LOCK_STATUS                   45
+#define PragTyp_PARSER_TRACE                  46
+#define PragTyp_STATS                         47

 /* Property flags associated with various pragma. */
 #define PragFlg_NeedSchema 0x01 /* Force schema load before running */
@@ -152,6 +153,11 @@ static const PragmaName aPragmaName[] = {
   /* ePragFlg:  */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1,
   /* ColNames:  */ 0, 0,
   /* iArg:      */ 0 },
+ {/* zName:     */ "auto_vacuum_slack_pages",
+  /* ePragTyp:  */ PragTyp_AUTO_VACUUM_SLACK_PAGES,
+  /* ePragFlg:  */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1,
+  /* ColNames:  */ 0, 0,
+  /* iArg:      */ 0 },
 #endif
 #if !defined(SQLITE_OMIT_FLAG_PRAGMAS)
 #if !defined(SQLITE_OMIT_AUTOMATIC_INDEX)
diff --git a/third_party/sqlite/src/tool/mkpragmatab.tcl b/third_party/sqlite/src/tool/mkpragmatab.tcl
index 83a85db9d6fd..6ed2dfc8e846 100644
--- a/third_party/sqlite/src/tool/mkpragmatab.tcl
+++ b/third_party/sqlite/src/tool/mkpragmatab.tcl
@@ -387,6 +387,10 @@ set pragma_def {
   TYPE: FLAG
   ARG:  SQLITE_LegacyAlter
   IF:   !defined(SQLITE_OMIT_FLAG_PRAGMAS)
+
+  NAME: auto_vacuum_slack_pages
+  FLAG: NeedSchema Result0 SchemaReq NoColumns1
+  IF:   !defined(SQLITE_OMIT_AUTOVACUUM)
 }

 # Open the output file
--
2.18.0

