diff options
Diffstat (limited to 'lib/libsqlite3')
66 files changed, 6578 insertions, 699 deletions
diff --git a/lib/libsqlite3/ext/fts3/fts3.c b/lib/libsqlite3/ext/fts3/fts3.c index 421052b9377..f80e303b56e 100644 --- a/lib/libsqlite3/ext/fts3/fts3.c +++ b/lib/libsqlite3/ext/fts3/fts3.c @@ -70,7 +70,7 @@ ** A doclist is stored like this: ** ** array { -** varint docid; +** varint docid; (delta from previous doclist) ** array { (position list for column 0) ** varint position; (2 more than the delta from previous position) ** } @@ -101,8 +101,8 @@ ** at D signals the start of a new column; the 1 at E indicates that the ** new column is column number 1. There are two positions at 12 and 45 ** (14-2 and 35-2+12). The 0 at H indicate the end-of-document. The -** 234 at I is the next docid. It has one position 72 (72-2) and then -** terminates with the 0 at K. +** 234 at I is the delta to next docid (357). It has one position 70 +** (72-2) and then terminates with the 0 at K. ** ** A "position-list" is the list of positions for multiple columns for ** a single docid. A "column-list" is the set of positions for a single @@ -571,6 +571,18 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){ } /* +** Create the %_stat table if it does not already exist. +*/ +void sqlite3Fts3CreateStatTable(int *pRc, Fts3Table *p){ + fts3DbExec(pRc, p->db, + "CREATE TABLE IF NOT EXISTS %Q.'%q_stat'" + "(id INTEGER PRIMARY KEY, value BLOB);", + p->zDb, p->zName + ); + if( (*pRc)==SQLITE_OK ) p->bHasStat = 1; +} + +/* ** Create the backing store tables (%_content, %_segments and %_segdir) ** required by the FTS3 table passed as the only argument. This is done ** as part of the vtab xCreate() method. @@ -630,11 +642,9 @@ static int fts3CreateTables(Fts3Table *p){ p->zDb, p->zName ); } + assert( p->bHasStat==p->bFts4 ); if( p->bHasStat ){ - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_stat'(id INTEGER PRIMARY KEY, value BLOB);", - p->zDb, p->zName - ); + sqlite3Fts3CreateStatTable(&rc, p); } return rc; } @@ -1275,7 +1285,9 @@ static int fts3InitVtab( p->nMaxPendingData = FTS3_MAX_PENDING_DATA; p->bHasDocsize = (isFts4 && bNoDocsize==0); p->bHasStat = isFts4; + p->bFts4 = isFts4; p->bDescIdx = bDescIdx; + p->bAutoincrmerge = 0xff; /* 0xff means setting unknown */ p->zContentTbl = zContent; p->zLanguageid = zLanguageid; zContent = 0; @@ -1328,6 +1340,16 @@ static int fts3InitVtab( rc = fts3CreateTables(p); } + /* Check to see if a legacy fts3 table has been "upgraded" by the + ** addition of a %_stat table so that it can use incremental merge. + */ + if( !isFts4 && !isCreate ){ + int rc2 = SQLITE_OK; + fts3DbExec(&rc2, db, "SELECT 1 FROM %Q.'%q_stat' WHERE id=2", + p->zDb, p->zName); + if( rc2==SQLITE_OK ) p->bHasStat = 1; + } + /* Figure out the page-size for the database. This is required in order to ** estimate the cost of loading large doclists from the database. */ fts3DatabasePageSize(&rc, p); @@ -2671,7 +2693,7 @@ static int fts3SegReaderCursor( */ int sqlite3Fts3SegReaderCursor( Fts3Table *p, /* FTS3 table handle */ - int iLangid, + int iLangid, /* Language-id to search */ int iIndex, /* Index to search (from 0 to p->nIndex-1) */ int iLevel, /* Level of segments to scan */ const char *zTerm, /* Term to query for */ @@ -2689,12 +2711,7 @@ int sqlite3Fts3SegReaderCursor( assert( FTS3_SEGCURSOR_ALL<0 && FTS3_SEGCURSOR_PENDING<0 ); assert( isPrefix==0 || isScan==0 ); - /* "isScan" is only set to true by the ft4aux module, an ordinary - ** full-text tables. */ - assert( isScan==0 || p->aIndex==0 ); - memset(pCsr, 0, sizeof(Fts3MultiSegReader)); - return fts3SegReaderCursor( p, iLangid, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr ); @@ -2959,7 +2976,7 @@ static int fts3FilterMethod( if( nVal==2 ) pCsr->iLangid = sqlite3_value_int(apVal[1]); rc = sqlite3Fts3ExprParse(p->pTokenizer, pCsr->iLangid, - p->azColumn, p->bHasStat, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr + p->azColumn, p->bFts4, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr ); if( rc!=SQLITE_OK ){ if( rc==SQLITE_ERROR ){ @@ -3102,8 +3119,42 @@ static int fts3UpdateMethod( ** hash-table to the database. */ static int fts3SyncMethod(sqlite3_vtab *pVtab){ - int rc = sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab); - sqlite3Fts3SegmentsClose((Fts3Table *)pVtab); + + /* Following an incremental-merge operation, assuming that the input + ** segments are not completely consumed (the usual case), they are updated + ** in place to remove the entries that have already been merged. This + ** involves updating the leaf block that contains the smallest unmerged + ** entry and each block (if any) between the leaf and the root node. So + ** if the height of the input segment b-trees is N, and input segments + ** are merged eight at a time, updating the input segments at the end + ** of an incremental-merge requires writing (8*(1+N)) blocks. N is usually + ** small - often between 0 and 2. So the overhead of the incremental + ** merge is somewhere between 8 and 24 blocks. To avoid this overhead + ** dwarfing the actual productive work accomplished, the incremental merge + ** is only attempted if it will write at least 64 leaf blocks. Hence + ** nMinMerge. + ** + ** Of course, updating the input segments also involves deleting a bunch + ** of blocks from the segments table. But this is not considered overhead + ** as it would also be required by a crisis-merge that used the same input + ** segments. + */ + const u32 nMinMerge = 64; /* Minimum amount of incr-merge work to do */ + + Fts3Table *p = (Fts3Table*)pVtab; + int rc = sqlite3Fts3PendingTermsFlush(p); + + if( rc==SQLITE_OK && p->bAutoincrmerge==1 && p->nLeafAdd>(nMinMerge/16) ){ + int mxLevel = 0; /* Maximum relative level value in db */ + int A; /* Incr-merge parameter A */ + + rc = sqlite3Fts3MaxLevel(p, &mxLevel); + assert( rc==SQLITE_OK || mxLevel==0 ); + A = p->nLeafAdd * mxLevel; + A += (A/2); + if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, 8); + } + sqlite3Fts3SegmentsClose(p); return rc; } @@ -3111,13 +3162,14 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ ** Implementation of xBegin() method. This is a no-op. */ static int fts3BeginMethod(sqlite3_vtab *pVtab){ - TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); + Fts3Table *p = (Fts3Table*)pVtab; UNUSED_PARAMETER(pVtab); assert( p->pSegments==0 ); assert( p->nPendingData==0 ); assert( p->inTransaction!=1 ); TESTONLY( p->inTransaction = 1 ); TESTONLY( p->mxSavepoint = -1; ); + p->nLeafAdd = 0; return SQLITE_OK; } @@ -3412,11 +3464,15 @@ static int fts3RenameMethod( ** Flush the contents of the pending-terms table to disk. */ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ + int rc = SQLITE_OK; UNUSED_PARAMETER(iSavepoint); assert( ((Fts3Table *)pVtab)->inTransaction ); assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint ); TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); - return fts3SyncMethod(pVtab); + if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){ + rc = fts3SyncMethod(pVtab); + } + return rc; } /* @@ -3889,7 +3945,7 @@ void sqlite3Fts3DoclistPrev( int nDoclist, /* Length of aDoclist in bytes */ char **ppIter, /* IN/OUT: Iterator pointer */ sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */ - int *pnList, /* IN/OUT: List length pointer */ + int *pnList, /* OUT: List length pointer */ u8 *pbEof /* OUT: End-of-file flag */ ){ char *p = *ppIter; @@ -3937,6 +3993,41 @@ void sqlite3Fts3DoclistPrev( } /* +** Iterate forwards through a doclist. +*/ +void sqlite3Fts3DoclistNext( + int bDescIdx, /* True if the doclist is desc */ + char *aDoclist, /* Pointer to entire doclist */ + int nDoclist, /* Length of aDoclist in bytes */ + char **ppIter, /* IN/OUT: Iterator pointer */ + sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */ + u8 *pbEof /* OUT: End-of-file flag */ +){ + char *p = *ppIter; + + assert( nDoclist>0 ); + assert( *pbEof==0 ); + assert( p || *piDocid==0 ); + assert( !p || (p>=aDoclist && p<=&aDoclist[nDoclist]) ); + + if( p==0 ){ + p = aDoclist; + p += sqlite3Fts3GetVarint(p, piDocid); + }else{ + fts3PoslistCopy(0, &p); + if( p>=&aDoclist[nDoclist] ){ + *pbEof = 1; + }else{ + sqlite3_int64 iVar; + p += sqlite3Fts3GetVarint(p, &iVar); + *piDocid += ((bDescIdx ? -1 : 1) * iVar); + } + } + + *ppIter = p; +} + +/* ** Attempt to move the phrase iterator to point to the next matching docid. ** If an error occurs, return an SQLite error code. Otherwise, return ** SQLITE_OK. @@ -4331,7 +4422,7 @@ static int fts3EvalStart(Fts3Cursor *pCsr){ fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc); /* Determine which, if any, tokens in the expression should be deferred. */ - if( rc==SQLITE_OK && nToken>1 && pTab->bHasStat ){ + if( rc==SQLITE_OK && nToken>1 && pTab->bFts4 ){ Fts3TokenAndCost *aTC; Fts3Expr **apOr; aTC = (Fts3TokenAndCost *)sqlite3_malloc( @@ -5091,26 +5182,87 @@ int sqlite3Fts3EvalPhraseStats( ** This function works regardless of whether or not the phrase is deferred, ** incremental, or neither. */ -char *sqlite3Fts3EvalPhrasePoslist( +int sqlite3Fts3EvalPhrasePoslist( Fts3Cursor *pCsr, /* FTS3 cursor object */ Fts3Expr *pExpr, /* Phrase to return doclist for */ - int iCol /* Column to return position list for */ + int iCol, /* Column to return position list for */ + char **ppOut /* OUT: Pointer to position list */ ){ Fts3Phrase *pPhrase = pExpr->pPhrase; Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - char *pIter = pPhrase->doclist.pList; + char *pIter; int iThis; + sqlite3_int64 iDocid; + /* If this phrase is applies specifically to some column other than + ** column iCol, return a NULL pointer. */ + *ppOut = 0; assert( iCol>=0 && iCol<pTab->nColumn ); - if( !pIter - || pExpr->bEof - || pExpr->iDocid!=pCsr->iPrevId - || (pPhrase->iColumn<pTab->nColumn && pPhrase->iColumn!=iCol) - ){ - return 0; + if( (pPhrase->iColumn<pTab->nColumn && pPhrase->iColumn!=iCol) ){ + return SQLITE_OK; + } + + iDocid = pExpr->iDocid; + pIter = pPhrase->doclist.pList; + if( iDocid!=pCsr->iPrevId || pExpr->bEof ){ + int bDescDoclist = pTab->bDescIdx; /* For DOCID_CMP macro */ + int bOr = 0; + u8 bEof = 0; + Fts3Expr *p; + + /* Check if this phrase descends from an OR expression node. If not, + ** return NULL. Otherwise, the entry that corresponds to docid + ** pCsr->iPrevId may lie earlier in the doclist buffer. */ + for(p=pExpr->pParent; p; p=p->pParent){ + if( p->eType==FTSQUERY_OR ) bOr = 1; + } + if( bOr==0 ) return SQLITE_OK; + + /* This is the descendent of an OR node. In this case we cannot use + ** an incremental phrase. Load the entire doclist for the phrase + ** into memory in this case. */ + if( pPhrase->bIncr ){ + int rc = SQLITE_OK; + int bEofSave = pExpr->bEof; + fts3EvalRestart(pCsr, pExpr, &rc); + while( rc==SQLITE_OK && !pExpr->bEof ){ + fts3EvalNextRow(pCsr, pExpr, &rc); + if( bEofSave==0 && pExpr->iDocid==iDocid ) break; + } + pIter = pPhrase->doclist.pList; + assert( rc!=SQLITE_OK || pPhrase->bIncr==0 ); + if( rc!=SQLITE_OK ) return rc; + } + + if( pExpr->bEof ){ + pIter = 0; + iDocid = 0; + } + bEof = (pPhrase->doclist.nAll==0); + assert( bDescDoclist==0 || bDescDoclist==1 ); + assert( pCsr->bDesc==0 || pCsr->bDesc==1 ); + + if( pCsr->bDesc==bDescDoclist ){ + int dummy; + while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){ + sqlite3Fts3DoclistPrev( + bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll, + &pIter, &iDocid, &dummy, &bEof + ); + } + }else{ + while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){ + sqlite3Fts3DoclistNext( + bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll, + &pIter, &iDocid, &bEof + ); + } + } + + if( bEof || iDocid!=pCsr->iPrevId ) pIter = 0; } + if( pIter==0 ) return SQLITE_OK; - assert( pPhrase->doclist.nList>0 ); if( *pIter==0x01 ){ pIter++; pIter += sqlite3Fts3GetVarint32(pIter, &iThis); @@ -5124,7 +5276,8 @@ char *sqlite3Fts3EvalPhrasePoslist( pIter += sqlite3Fts3GetVarint32(pIter, &iThis); } - return ((iCol==iThis)?pIter:0); + *ppOut = ((iCol==iThis)?pIter:0); + return SQLITE_OK; } /* @@ -5147,6 +5300,7 @@ void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ } } + /* ** Return SQLITE_CORRUPT_VTAB. */ diff --git a/lib/libsqlite3/ext/fts3/fts3Int.h b/lib/libsqlite3/ext/fts3/fts3Int.h index 393cd6aea11..a60b4b6849b 100644 --- a/lib/libsqlite3/ext/fts3/fts3Int.h +++ b/lib/libsqlite3/ext/fts3/fts3Int.h @@ -67,6 +67,9 @@ extern const sqlite3_api_routines *sqlite3_api; #ifndef MIN # define MIN(x,y) ((x)<(y)?(x):(y)) #endif +#ifndef MAX +# define MAX(x,y) ((x)>(y)?(x):(y)) +#endif /* ** Maximum length of a varint encoded integer. The varint format is different @@ -121,7 +124,7 @@ extern const sqlite3_api_routines *sqlite3_api; # define NEVER(X) (0) #else # define ALWAYS(x) (x) -# define NEVER(X) (x) +# define NEVER(x) (x) #endif /* @@ -131,6 +134,7 @@ typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */ typedef short int i16; /* 2-byte (or larger) signed integer */ typedef unsigned int u32; /* 4-byte unsigned integer */ typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */ +typedef sqlite3_int64 i64; /* 8-byte signed integer */ /* ** Macro used to suppress compiler warnings for unused parameters. @@ -193,36 +197,43 @@ struct Fts3Table { sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ char *zContentTbl; /* content=xxx option, or NULL */ char *zLanguageid; /* languageid=xxx option, or NULL */ + u8 bAutoincrmerge; /* True if automerge=1 */ + u32 nLeafAdd; /* Number of leaf blocks added this trans */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ - sqlite3_stmt *aStmt[28]; + sqlite3_stmt *aStmt[37]; char *zReadExprlist; char *zWriteExprlist; int nNodeSize; /* Soft limit for node size */ + u8 bFts4; /* True for FTS4, false for FTS3 */ u8 bHasStat; /* True if %_stat table exists */ u8 bHasDocsize; /* True if %_docsize table exists */ u8 bDescIdx; /* True if doclists are in reverse order */ + u8 bIgnoreSavepoint; /* True to ignore xSavepoint invocations */ int nPgsz; /* Page size for host database */ char *zSegmentsTbl; /* Name of %_segments table */ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ - /* TODO: Fix the first paragraph of this comment. - ** + /* ** The following array of hash tables is used to buffer pending index - ** updates during transactions. Variable nPendingData estimates the memory - ** size of the pending data, including hash table overhead, not including - ** malloc overhead. When nPendingData exceeds nMaxPendingData, the buffer - ** is flushed automatically. Variable iPrevDocid is the docid of the most - ** recently inserted record. + ** updates during transactions. All pending updates buffered at any one + ** time must share a common language-id (see the FTS4 langid= feature). + ** The current language id is stored in variable iPrevLangid. ** ** A single FTS4 table may have multiple full-text indexes. For each index ** there is an entry in the aIndex[] array. Index 0 is an index of all the ** terms that appear in the document set. Each subsequent index in aIndex[] ** is an index of prefixes of a specific length. + ** + ** Variable nPendingData contains an estimate the memory consumed by the + ** pending data structures, including hash table overhead, but not including + ** malloc overhead. When nPendingData exceeds nMaxPendingData, all hash + ** tables are flushed to disk. Variable iPrevDocid is the docid of the most + ** recently inserted record. */ int nIndex; /* Size of aIndex[] */ struct Fts3Index { @@ -421,6 +432,7 @@ int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); void sqlite3Fts3SegmentsClose(Fts3Table *); +int sqlite3Fts3MaxLevel(Fts3Table *, int *); /* Special values interpreted by sqlite3SegReaderCursor() */ #define FTS3_SEGCURSOR_PENDING -1 @@ -472,6 +484,8 @@ struct Fts3MultiSegReader { int nDoclist; /* Size of aDoclist[] in bytes */ }; +int sqlite3Fts3Incrmerge(Fts3Table*,int,int); + /* fts3.c */ int sqlite3Fts3PutVarint(char *, sqlite3_int64); int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); @@ -481,6 +495,7 @@ void sqlite3Fts3Dequote(char *); void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *); +void sqlite3Fts3CreateStatTable(int*, Fts3Table*); /* fts3_tokenizer.c */ const char *sqlite3Fts3NextToken(const char *, int *); @@ -520,7 +535,7 @@ int sqlite3Fts3MsrIncrStart( Fts3Table*, Fts3MultiSegReader*, int, const char*, int); int sqlite3Fts3MsrIncrNext( Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); -char *sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol); +int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **); int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); diff --git a/lib/libsqlite3/ext/fts3/fts3_icu.c b/lib/libsqlite3/ext/fts3/fts3_icu.c index a10a55d67b9..5e9c900b091 100644 --- a/lib/libsqlite3/ext/fts3/fts3_icu.c +++ b/lib/libsqlite3/ext/fts3/fts3_icu.c @@ -110,7 +110,10 @@ static int icuOpen( *ppCursor = 0; - if( nInput<0 ){ + if( zInput==0 ){ + nInput = 0; + zInput = ""; + }else if( nInput<0 ){ nInput = strlen(zInput); } nChar = nInput+1; diff --git a/lib/libsqlite3/ext/fts3/fts3_snippet.c b/lib/libsqlite3/ext/fts3/fts3_snippet.c index fd5bc9786b5..6fce3d0a691 100644 --- a/lib/libsqlite3/ext/fts3/fts3_snippet.c +++ b/lib/libsqlite3/ext/fts3/fts3_snippet.c @@ -360,10 +360,11 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ SnippetIter *p = (SnippetIter *)ctx; SnippetPhrase *pPhrase = &p->aPhrase[iPhrase]; char *pCsr; + int rc; pPhrase->nToken = pExpr->pPhrase->nToken; - - pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol); + rc = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol, &pCsr); + assert( rc==SQLITE_OK || pCsr==0 ); if( pCsr ){ int iFirst = 0; pPhrase->pList = pCsr; @@ -374,10 +375,12 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ pPhrase->iHead = iFirst; pPhrase->iTail = iFirst; }else{ - assert( pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 ); + assert( rc!=SQLITE_OK || ( + pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 + )); } - return SQLITE_OK; + return rc; } /* @@ -770,13 +773,14 @@ static int fts3ExprLocalHitsCb( int iPhrase, /* Phrase number */ void *pCtx /* Pointer to MatchInfo structure */ ){ + int rc = SQLITE_OK; MatchInfo *p = (MatchInfo *)pCtx; int iStart = iPhrase * p->nCol * 3; int i; - for(i=0; i<p->nCol; i++){ + for(i=0; i<p->nCol && rc==SQLITE_OK; i++){ char *pCsr; - pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i); + rc = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i, &pCsr); if( pCsr ){ p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr); }else{ @@ -784,7 +788,7 @@ static int fts3ExprLocalHitsCb( } } - return SQLITE_OK; + return rc; } static int fts3MatchinfoCheck( @@ -794,8 +798,8 @@ static int fts3MatchinfoCheck( ){ if( (cArg==FTS3_MATCHINFO_NPHRASE) || (cArg==FTS3_MATCHINFO_NCOL) - || (cArg==FTS3_MATCHINFO_NDOC && pTab->bHasStat) - || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bHasStat) + || (cArg==FTS3_MATCHINFO_NDOC && pTab->bFts4) + || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bFts4) || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize) || (cArg==FTS3_MATCHINFO_LCS) || (cArg==FTS3_MATCHINFO_HITS) @@ -945,8 +949,10 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ int nLive = 0; /* Number of iterators in aIter not at EOF */ for(i=0; i<pInfo->nPhrase; i++){ + int rc; LcsIterator *pIt = &aIter[i]; - pIt->pRead = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol); + rc = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol, &pIt->pRead); + if( rc!=SQLITE_OK ) return rc; if( pIt->pRead ){ pIt->iPos = pIt->iPosOffset; fts3LcsIteratorAdvance(&aIter[i]); @@ -1298,9 +1304,10 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ int iTerm; /* For looping through nTerm phrase terms */ char *pList; /* Pointer to position list for phrase */ int iPos = 0; /* First position in position-list */ + int rc; UNUSED_PARAMETER(iPhrase); - pList = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol); + rc = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol, &pList); nTerm = pExpr->pPhrase->nToken; if( pList ){ fts3GetDeltaPosition(&pList, &iPos); @@ -1314,7 +1321,7 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ pT->iPos = iPos; } - return SQLITE_OK; + return rc; } /* diff --git a/lib/libsqlite3/ext/fts3/fts3_term.c b/lib/libsqlite3/ext/fts3/fts3_term.c index 2108fc12519..c49d5cb65d7 100644 --- a/lib/libsqlite3/ext/fts3/fts3_term.c +++ b/lib/libsqlite3/ext/fts3/fts3_term.c @@ -73,6 +73,7 @@ static int fts3termConnectMethod( Fts3termTable *p; /* Virtual table object to return */ int iIndex = 0; + UNUSED_PARAMETER(pCtx); if( argc==5 ){ iIndex = atoi(argv[4]); argc--; @@ -87,9 +88,9 @@ static int fts3termConnectMethod( } zDb = argv[1]; - nDb = strlen(zDb); + nDb = (int)strlen(zDb); zFts3 = argv[3]; - nFts3 = strlen(zFts3); + nFts3 = (int)strlen(zFts3); rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA); if( rc!=SQLITE_OK ) return rc; @@ -231,12 +232,12 @@ static int fts3termNextMethod(sqlite3_vtab_cursor *pCursor){ if( v==1 ){ pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v); - pCsr->iCol += v; + pCsr->iCol += (int)v; pCsr->iPos = 0; pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v); } - pCsr->iPos += (v - 2); + pCsr->iPos += (int)(v - 2); return SQLITE_OK; } @@ -357,7 +358,10 @@ int sqlite3Fts3InitTerm(sqlite3 *db){ 0, /* xCommit */ 0, /* xRollback */ 0, /* xFindFunction */ - 0 /* xRename */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ }; int rc; /* Return code */ diff --git a/lib/libsqlite3/ext/fts3/fts3_test.c b/lib/libsqlite3/ext/fts3/fts3_test.c index 01873d4978b..4da0b8f13d2 100644 --- a/lib/libsqlite3/ext/fts3/fts3_test.c +++ b/lib/libsqlite3/ext/fts3/fts3_test.c @@ -22,7 +22,8 @@ #include <string.h> #include <assert.h> -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) +#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) /* Required so that the "ifdef SQLITE_ENABLE_FTS3" below works */ #include "fts3Int.h" @@ -161,6 +162,8 @@ static int fts3_near_match_cmd( Tcl_Obj **apExprToken; int nExprToken; + UNUSED_PARAMETER(clientData); + /* Must have 3 or more arguments. */ if( objc<3 || (objc%2)==0 ){ Tcl_WrongNumArgs(interp, 1, objv, "DOCUMENT EXPR ?OPTION VALUE?..."); @@ -314,6 +317,7 @@ static int fts3_configure_incr_load_cmd( Tcl_SetObjResult(interp, pRet); Tcl_DecrRefCount(pRet); #endif + UNUSED_PARAMETER(clientData); return TCL_OK; } @@ -352,6 +356,8 @@ static int testTokenizerCreate( sqlite3_tokenizer **ppTokenizer ){ test_tokenizer *pNew; + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); pNew = sqlite3_malloc(sizeof(test_tokenizer)); if( !pNew ) return SQLITE_NOMEM; @@ -384,7 +390,7 @@ static int testTokenizerOpen( memset(pCsr, 0, sizeof(test_tokenizer_cursor)); pCsr->aInput = pInput; if( nBytes<0 ){ - pCsr->nInput = strlen(pInput); + pCsr->nInput = (int)strlen(pInput); }else{ pCsr->nInput = nBytes; } @@ -437,7 +443,7 @@ static int testTokenizerNext( const char *pToken = p; int nToken; while( p<pEnd && testIsTokenChar(*p) ) p++; - nToken = p-pToken; + nToken = (int)(p-pToken); /* Copy the token into the buffer */ if( nToken>pCsr->nBuffer ){ @@ -455,12 +461,12 @@ static int testTokenizerNext( for(i=0; i<nToken; i++) pCsr->aBuffer[i] = testTolower(pToken[i]); } pCsr->iToken++; - pCsr->iInput = p - pCsr->aInput; + pCsr->iInput = (int)(p - pCsr->aInput); *ppToken = pCsr->aBuffer; *pnBytes = nToken; - *piStartOffset = pToken - pCsr->aInput; - *piEndOffset = p - pCsr->aInput; + *piStartOffset = (int)(pToken - pCsr->aInput); + *piEndOffset = (int)(p - pCsr->aInput); *piPosition = pCsr->iToken; } } @@ -507,6 +513,7 @@ static int fts3_test_tokenizer_cmd( (const unsigned char *)&pPtr, sizeof(sqlite3_tokenizer_module *) )); #endif + UNUSED_PARAMETER(clientData); return TCL_OK; } @@ -524,4 +531,5 @@ int Sqlitetestfts3_Init(Tcl_Interp *interp){ ); return TCL_OK; } +#endif /* SQLITE_ENABLE_FTS3 || SQLITE_ENABLE_FTS4 */ #endif /* ifdef SQLITE_TEST */ diff --git a/lib/libsqlite3/ext/fts3/fts3_write.c b/lib/libsqlite3/ext/fts3/fts3_write.c index bc20dfa3f34..fa5fb02451a 100644 --- a/lib/libsqlite3/ext/fts3/fts3_write.c +++ b/lib/libsqlite3/ext/fts3/fts3_write.c @@ -24,6 +24,9 @@ #include <assert.h> #include <stdlib.h> + +#define FTS_MAX_APPENDABLE_HEIGHT 16 + /* ** When full-text index nodes are loaded from disk, the buffer that they ** are loaded into has the following number of bytes of padding at the end @@ -63,6 +66,29 @@ int test_fts3_node_chunk_threshold = (4*1024)*4; # define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) #endif +/* +** The two values that may be meaningfully bound to the :1 parameter in +** statements SQL_REPLACE_STAT and SQL_SELECT_STAT. +*/ +#define FTS_STAT_DOCTOTAL 0 +#define FTS_STAT_INCRMERGEHINT 1 +#define FTS_STAT_AUTOINCRMERGE 2 + +/* +** If FTS_LOG_MERGES is defined, call sqlite3_log() to report each automatic +** and incremental merge operation that takes place. This is used for +** debugging FTS only, it should not usually be turned on in production +** systems. +*/ +#ifdef FTS3_LOG_MERGES +static void fts3LogMerge(int nMerge, sqlite3_int64 iAbsLevel){ + sqlite3_log(SQLITE_OK, "%d-way merge from level %d", nMerge, (int)iAbsLevel); +} +#else +#define fts3LogMerge(x, y) +#endif + + typedef struct PendingList PendingList; typedef struct SegmentNode SegmentNode; typedef struct SegmentWriter SegmentWriter; @@ -224,15 +250,22 @@ struct SegmentNode { #define SQL_DELETE_DOCSIZE 19 #define SQL_REPLACE_DOCSIZE 20 #define SQL_SELECT_DOCSIZE 21 -#define SQL_SELECT_DOCTOTAL 22 -#define SQL_REPLACE_DOCTOTAL 23 +#define SQL_SELECT_STAT 22 +#define SQL_REPLACE_STAT 23 #define SQL_SELECT_ALL_PREFIX_LEVEL 24 #define SQL_DELETE_ALL_TERMS_SEGDIR 25 - #define SQL_DELETE_SEGDIR_RANGE 26 - #define SQL_SELECT_ALL_LANGID 27 +#define SQL_FIND_MERGE_LEVEL 28 +#define SQL_MAX_LEAF_NODE_ESTIMATE 29 +#define SQL_DELETE_SEGDIR_ENTRY 30 +#define SQL_SHIFT_SEGDIR_ENTRY 31 +#define SQL_SELECT_SEGDIR 32 +#define SQL_CHOMP_SEGDIR 33 +#define SQL_SEGMENT_IS_APPENDABLE 34 +#define SQL_SELECT_INDEXES 35 +#define SQL_SELECT_MXLEVEL 36 /* ** This function is used to obtain an SQLite prepared statement handle @@ -261,9 +294,9 @@ static int fts3SqlStmt( /* 6 */ "DELETE FROM %Q.'%q_stat'", /* 7 */ "SELECT %s WHERE rowid=?", /* 8 */ "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1", -/* 9 */ "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", +/* 9 */ "REPLACE INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", /* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)", -/* 11 */ "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", +/* 11 */ "REPLACE INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", /* Return segments in order from oldest to newest.*/ /* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " @@ -281,14 +314,61 @@ static int fts3SqlStmt( /* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", /* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", /* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", -/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", -/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", +/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=?", +/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(?,?)", /* 24 */ "", /* 25 */ "", /* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", /* 27 */ "SELECT DISTINCT level / (1024 * ?) FROM %Q.'%q_segdir'", +/* This statement is used to determine which level to read the input from +** when performing an incremental merge. It returns the absolute level number +** of the oldest level in the db that contains at least ? segments. Or, +** if no level in the FTS index contains more than ? segments, the statement +** returns zero rows. */ +/* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?" + " ORDER BY (level %% 1024) ASC LIMIT 1", + +/* Estimate the upper limit on the number of leaf nodes in a new segment +** created by merging the oldest :2 segments from absolute level :1. See +** function sqlite3Fts3Incrmerge() for details. */ +/* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) " + " FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?", + +/* SQL_DELETE_SEGDIR_ENTRY +** Delete the %_segdir entry on absolute level :1 with index :2. */ +/* 30 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", + +/* SQL_SHIFT_SEGDIR_ENTRY +** Modify the idx value for the segment with idx=:3 on absolute level :2 +** to :1. */ +/* 31 */ "UPDATE %Q.'%q_segdir' SET idx = ? WHERE level=? AND idx=?", + +/* SQL_SELECT_SEGDIR +** Read a single entry from the %_segdir table. The entry from absolute +** level :1 with index value :2. */ +/* 32 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", + +/* SQL_CHOMP_SEGDIR +** Update the start_block (:1) and root (:2) fields of the %_segdir +** entry located on absolute level :3 with index :4. */ +/* 33 */ "UPDATE %Q.'%q_segdir' SET start_block = ?, root = ?" + "WHERE level = ? AND idx = ?", + +/* SQL_SEGMENT_IS_APPENDABLE +** Return a single row if the segment with end_block=? is appendable. Or +** no rows otherwise. */ +/* 34 */ "SELECT 1 FROM %Q.'%q_segments' WHERE blockid=? AND block IS NULL", + +/* SQL_SELECT_INDEXES +** Return the list of valid segment indexes for absolute level ? */ +/* 35 */ "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC", + +/* SQL_SELECT_MXLEVEL +** Return the largest relative level in the FTS index or indexes. */ +/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'" }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; @@ -326,22 +406,18 @@ static int fts3SqlStmt( return rc; } + static int fts3SelectDocsize( Fts3Table *pTab, /* FTS3 table handle */ - int eStmt, /* Either SQL_SELECT_DOCSIZE or DOCTOTAL */ sqlite3_int64 iDocid, /* Docid to bind for SQL_SELECT_DOCSIZE */ sqlite3_stmt **ppStmt /* OUT: Statement handle */ ){ sqlite3_stmt *pStmt = 0; /* Statement requested from fts3SqlStmt() */ int rc; /* Return code */ - assert( eStmt==SQL_SELECT_DOCSIZE || eStmt==SQL_SELECT_DOCTOTAL ); - - rc = fts3SqlStmt(pTab, eStmt, &pStmt, 0); + rc = fts3SqlStmt(pTab, SQL_SELECT_DOCSIZE, &pStmt, 0); if( rc==SQLITE_OK ){ - if( eStmt==SQL_SELECT_DOCSIZE ){ - sqlite3_bind_int64(pStmt, 1, iDocid); - } + sqlite3_bind_int64(pStmt, 1, iDocid); rc = sqlite3_step(pStmt); if( rc!=SQLITE_ROW || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){ rc = sqlite3_reset(pStmt); @@ -360,7 +436,21 @@ int sqlite3Fts3SelectDoctotal( Fts3Table *pTab, /* Fts3 table handle */ sqlite3_stmt **ppStmt /* OUT: Statement handle */ ){ - return fts3SelectDocsize(pTab, SQL_SELECT_DOCTOTAL, 0, ppStmt); + sqlite3_stmt *pStmt = 0; + int rc; + rc = fts3SqlStmt(pTab, SQL_SELECT_STAT, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); + if( sqlite3_step(pStmt)!=SQLITE_ROW + || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB + ){ + rc = sqlite3_reset(pStmt); + if( rc==SQLITE_OK ) rc = FTS_CORRUPT_VTAB; + pStmt = 0; + } + } + *ppStmt = pStmt; + return rc; } int sqlite3Fts3SelectDocsize( @@ -368,7 +458,7 @@ int sqlite3Fts3SelectDocsize( sqlite3_int64 iDocid, /* Docid to read size data for */ sqlite3_stmt **ppStmt /* OUT: Statement handle */ ){ - return fts3SelectDocsize(pTab, SQL_SELECT_DOCSIZE, iDocid, ppStmt); + return fts3SelectDocsize(pTab, iDocid, ppStmt); } /* @@ -458,10 +548,10 @@ int sqlite3Fts3ReadLock(Fts3Table *p){ ** iIndex starts at absolute level ((iLangid * (nPrefix+1) + iIndex) * 1024). */ static sqlite3_int64 getAbsoluteLevel( - Fts3Table *p, - int iLangid, - int iIndex, - int iLevel + Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language id */ + int iIndex, /* Index in p->aIndex[] */ + int iLevel /* Level of segments */ ){ sqlite3_int64 iBase; /* First absolute level for iLangid/iIndex */ assert( iLangid>=0 ); @@ -472,7 +562,6 @@ static sqlite3_int64 getAbsoluteLevel( return iBase + iLevel; } - /* ** Set *ppStmt to a statement handle that may be used to iterate through ** all rows in the %_segdir table, from oldest to newest. If successful, @@ -1038,6 +1127,7 @@ static int fts3AllocateSegdirIdx( ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. */ if( iNext>=FTS3_MERGE_COUNT ){ + fts3LogMerge(16, getAbsoluteLevel(p, iLangid, iIndex, iLevel)); rc = fts3SegmentMerge(p, iLangid, iIndex, iLevel); *piIdx = 0; }else{ @@ -1085,7 +1175,7 @@ int sqlite3Fts3ReadBlock( int rc; /* Return code */ /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */ - assert( pnBlob); + assert( pnBlob ); if( p->pSegments ){ rc = sqlite3_blob_reopen(p->pSegments, iBlockid); @@ -1426,7 +1516,7 @@ int sqlite3Fts3MsrOvfl( int rc = SQLITE_OK; int pgsz = p->nPgsz; - assert( p->bHasStat ); + assert( p->bFts4 ); assert( pgsz>0 ); for(ii=0; rc==SQLITE_OK && ii<pMsr->nSegment; ii++){ @@ -1784,6 +1874,27 @@ static int fts3WriteSegment( return rc; } +/* +** Find the largest relative level number in the table. If successful, set +** *pnMax to this value and return SQLITE_OK. Otherwise, if an error occurs, +** set *pnMax to zero and return an SQLite error code. +*/ +int sqlite3Fts3MaxLevel(Fts3Table *p, int *pnMax){ + int rc; + int mxLevel = 0; + sqlite3_stmt *pStmt = 0; + + rc = fts3SqlStmt(p, SQL_SELECT_MXLEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + mxLevel = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_reset(pStmt); + } + *pnMax = mxLevel; + return rc; +} + /* ** Insert a record into the %_segdir table. */ @@ -2100,6 +2211,7 @@ static int fts3SegWriterAdd( /* The current leaf node is full. Write it out to the database. */ rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData); if( rc!=SQLITE_OK ) return rc; + p->nLeafAdd++; /* Add the current term to the interior node tree. The term added to ** the interior tree must: @@ -2208,6 +2320,7 @@ static int fts3SegWriterFlush( rc = fts3WriteSegdir( p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData); } + p->nLeafAdd++; return rc; } @@ -2290,6 +2403,29 @@ static int fts3SegmentMaxLevel( } /* +** Delete all entries in the %_segments table associated with the segment +** opened with seg-reader pSeg. This function does not affect the contents +** of the %_segdir table. +*/ +static int fts3DeleteSegment( + Fts3Table *p, /* FTS table handle */ + Fts3SegReader *pSeg /* Segment to delete */ +){ + int rc = SQLITE_OK; /* Return code */ + if( pSeg->iStartBlock ){ + sqlite3_stmt *pDelete; /* SQL statement to delete rows */ + rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDelete, 1, pSeg->iStartBlock); + sqlite3_bind_int64(pDelete, 2, pSeg->iEndBlock); + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); + } + } + return rc; +} + +/* ** This function is used after merging multiple segments into a single large ** segment to delete the old, now redundant, segment b-trees. Specifically, ** it: @@ -2311,19 +2447,12 @@ static int fts3DeleteSegdir( Fts3SegReader **apSegment, /* Array of SegReader objects */ int nReader /* Size of array apSegment */ ){ - int rc; /* Return Code */ + int rc = SQLITE_OK; /* Return Code */ int i; /* Iterator variable */ - sqlite3_stmt *pDelete; /* SQL statement to delete rows */ + sqlite3_stmt *pDelete = 0; /* SQL statement to delete rows */ - rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0); for(i=0; rc==SQLITE_OK && i<nReader; i++){ - Fts3SegReader *pSegment = apSegment[i]; - if( pSegment->iStartBlock ){ - sqlite3_bind_int64(pDelete, 1, pSegment->iStartBlock); - sqlite3_bind_int64(pDelete, 2, pSegment->iEndBlock); - sqlite3_step(pDelete); - rc = sqlite3_reset(pDelete); - } + rc = fts3DeleteSegment(p, apSegment[i]); } if( rc!=SQLITE_OK ){ return rc; @@ -2899,11 +3028,28 @@ static int fts3SegmentMerge( int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ int rc = SQLITE_OK; int i; + for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){ rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING); if( rc==SQLITE_DONE ) rc = SQLITE_OK; } sqlite3Fts3PendingTermsClear(p); + + /* Determine the auto-incr-merge setting if unknown. If enabled, + ** estimate the number of leaf blocks of content to be written + */ + if( rc==SQLITE_OK && p->bHasStat + && p->bAutoincrmerge==0xff && p->nLeafAdd>0 + ){ + sqlite3_stmt *pStmt = 0; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); + rc = sqlite3_step(pStmt); + p->bAutoincrmerge = (rc==SQLITE_ROW && sqlite3_column_int(pStmt, 0)); + rc = sqlite3_reset(pStmt); + } + } return rc; } @@ -3014,12 +3160,13 @@ static void fts3UpdateDocTotals( return; } pBlob = (char*)&a[nStat]; - rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0); + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); if( rc ){ sqlite3_free(a); *pRC = rc; return; } + sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); if( sqlite3_step(pStmt)==SQLITE_ROW ){ fts3DecodeIntArray(nStat, a, sqlite3_column_blob(pStmt, 0), @@ -3043,13 +3190,14 @@ static void fts3UpdateDocTotals( a[i+1] = x; } fts3EncodeIntArray(nStat, a, pBlob, &nBlob); - rc = fts3SqlStmt(p, SQL_REPLACE_DOCTOTAL, &pStmt, 0); + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); if( rc ){ sqlite3_free(a); *pRC = rc; return; } - sqlite3_bind_blob(pStmt, 1, pBlob, nBlob, SQLITE_STATIC); + sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); + sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC); sqlite3_step(pStmt); *pRC = sqlite3_reset(pStmt); sqlite3_free(a); @@ -3154,7 +3302,7 @@ static int fts3DoRebuild(Fts3Table *p){ } } } - if( p->bHasStat ){ + if( p->bFts4 ){ fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nEntry); } sqlite3_free(aSz); @@ -3170,6 +3318,1687 @@ static int fts3DoRebuild(Fts3Table *p){ return rc; } + +/* +** This function opens a cursor used to read the input data for an +** incremental merge operation. Specifically, it opens a cursor to scan +** the oldest nSeg segments (idx=0 through idx=(nSeg-1)) in absolute +** level iAbsLevel. +*/ +static int fts3IncrmergeCsr( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level to open */ + int nSeg, /* Number of segments to merge */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ +){ + int rc; /* Return Code */ + sqlite3_stmt *pStmt = 0; /* Statement used to read %_segdir entry */ + int nByte; /* Bytes allocated at pCsr->apSegment[] */ + + /* Allocate space for the Fts3MultiSegReader.aCsr[] array */ + memset(pCsr, 0, sizeof(*pCsr)); + nByte = sizeof(Fts3SegReader *) * nSeg; + pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte); + + if( pCsr->apSegment==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pCsr->apSegment, 0, nByte); + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); + } + if( rc==SQLITE_OK ){ + int i; + int rc2; + sqlite3_bind_int64(pStmt, 1, iAbsLevel); + assert( pCsr->nSegment==0 ); + for(i=0; rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW && i<nSeg; i++){ + rc = sqlite3Fts3SegReaderNew(i, 0, + sqlite3_column_int64(pStmt, 1), /* segdir.start_block */ + sqlite3_column_int64(pStmt, 2), /* segdir.leaves_end_block */ + sqlite3_column_int64(pStmt, 3), /* segdir.end_block */ + sqlite3_column_blob(pStmt, 4), /* segdir.root */ + sqlite3_column_bytes(pStmt, 4), /* segdir.root */ + &pCsr->apSegment[i] + ); + pCsr->nSegment++; + } + rc2 = sqlite3_reset(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + } + + return rc; +} + +typedef struct IncrmergeWriter IncrmergeWriter; +typedef struct NodeWriter NodeWriter; +typedef struct Blob Blob; +typedef struct NodeReader NodeReader; + +/* +** An instance of the following structure is used as a dynamic buffer +** to build up nodes or other blobs of data in. +** +** The function blobGrowBuffer() is used to extend the allocation. +*/ +struct Blob { + char *a; /* Pointer to allocation */ + int n; /* Number of valid bytes of data in a[] */ + int nAlloc; /* Allocated size of a[] (nAlloc>=n) */ +}; + +/* +** This structure is used to build up buffers containing segment b-tree +** nodes (blocks). +*/ +struct NodeWriter { + sqlite3_int64 iBlock; /* Current block id */ + Blob key; /* Last key written to the current block */ + Blob block; /* Current block image */ +}; + +/* +** An object of this type contains the state required to create or append +** to an appendable b-tree segment. +*/ +struct IncrmergeWriter { + int nLeafEst; /* Space allocated for leaf blocks */ + int nWork; /* Number of leaf pages flushed */ + sqlite3_int64 iAbsLevel; /* Absolute level of input segments */ + int iIdx; /* Index of *output* segment in iAbsLevel+1 */ + sqlite3_int64 iStart; /* Block number of first allocated block */ + sqlite3_int64 iEnd; /* Block number of last allocated block */ + NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT]; +}; + +/* +** An object of the following type is used to read data from a single +** FTS segment node. See the following functions: +** +** nodeReaderInit() +** nodeReaderNext() +** nodeReaderRelease() +*/ +struct NodeReader { + const char *aNode; + int nNode; + int iOff; /* Current offset within aNode[] */ + + /* Output variables. Containing the current node entry. */ + sqlite3_int64 iChild; /* Pointer to child node */ + Blob term; /* Current term */ + const char *aDoclist; /* Pointer to doclist */ + int nDoclist; /* Size of doclist in bytes */ +}; + +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, if the allocation at pBlob->a is not already at least nMin +** bytes in size, extend (realloc) it to be so. +** +** If an OOM error occurs, set *pRc to SQLITE_NOMEM and leave pBlob->a +** unmodified. Otherwise, if the allocation succeeds, update pBlob->nAlloc +** to reflect the new size of the pBlob->a[] buffer. +*/ +static void blobGrowBuffer(Blob *pBlob, int nMin, int *pRc){ + if( *pRc==SQLITE_OK && nMin>pBlob->nAlloc ){ + int nAlloc = nMin; + char *a = (char *)sqlite3_realloc(pBlob->a, nAlloc); + if( a ){ + pBlob->nAlloc = nAlloc; + pBlob->a = a; + }else{ + *pRc = SQLITE_NOMEM; + } + } +} + +/* +** Attempt to advance the node-reader object passed as the first argument to +** the next entry on the node. +** +** Return an error code if an error occurs (SQLITE_NOMEM is possible). +** Otherwise return SQLITE_OK. If there is no next entry on the node +** (e.g. because the current entry is the last) set NodeReader->aNode to +** NULL to indicate EOF. Otherwise, populate the NodeReader structure output +** variables for the new entry. +*/ +static int nodeReaderNext(NodeReader *p){ + int bFirst = (p->term.n==0); /* True for first term on the node */ + int nPrefix = 0; /* Bytes to copy from previous term */ + int nSuffix = 0; /* Bytes to append to the prefix */ + int rc = SQLITE_OK; /* Return code */ + + assert( p->aNode ); + if( p->iChild && bFirst==0 ) p->iChild++; + if( p->iOff>=p->nNode ){ + /* EOF */ + p->aNode = 0; + }else{ + if( bFirst==0 ){ + p->iOff += sqlite3Fts3GetVarint32(&p->aNode[p->iOff], &nPrefix); + } + p->iOff += sqlite3Fts3GetVarint32(&p->aNode[p->iOff], &nSuffix); + + blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc); + if( rc==SQLITE_OK ){ + memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix); + p->term.n = nPrefix+nSuffix; + p->iOff += nSuffix; + if( p->iChild==0 ){ + p->iOff += sqlite3Fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist); + p->aDoclist = &p->aNode[p->iOff]; + p->iOff += p->nDoclist; + } + } + } + + assert( p->iOff<=p->nNode ); + + return rc; +} + +/* +** Release all dynamic resources held by node-reader object *p. +*/ +static void nodeReaderRelease(NodeReader *p){ + sqlite3_free(p->term.a); +} + +/* +** Initialize a node-reader object to read the node in buffer aNode/nNode. +** +** If successful, SQLITE_OK is returned and the NodeReader object set to +** point to the first entry on the node (if any). Otherwise, an SQLite +** error code is returned. +*/ +static int nodeReaderInit(NodeReader *p, const char *aNode, int nNode){ + memset(p, 0, sizeof(NodeReader)); + p->aNode = aNode; + p->nNode = nNode; + + /* Figure out if this is a leaf or an internal node. */ + if( p->aNode[0] ){ + /* An internal node. */ + p->iOff = 1 + sqlite3Fts3GetVarint(&p->aNode[1], &p->iChild); + }else{ + p->iOff = 1; + } + + return nodeReaderNext(p); +} + +/* +** This function is called while writing an FTS segment each time a leaf o +** node is finished and written to disk. The key (zTerm/nTerm) is guaranteed +** to be greater than the largest key on the node just written, but smaller +** than or equal to the first key that will be written to the next leaf +** node. +** +** The block id of the leaf node just written to disk may be found in +** (pWriter->aNodeWriter[0].iBlock) when this function is called. +*/ +static int fts3IncrmergePush( + Fts3Table *p, /* Fts3 table handle */ + IncrmergeWriter *pWriter, /* Writer object */ + const char *zTerm, /* Term to write to internal node */ + int nTerm /* Bytes at zTerm */ +){ + sqlite3_int64 iPtr = pWriter->aNodeWriter[0].iBlock; + int iLayer; + + assert( nTerm>0 ); + for(iLayer=1; ALWAYS(iLayer<FTS_MAX_APPENDABLE_HEIGHT); iLayer++){ + sqlite3_int64 iNextPtr = 0; + NodeWriter *pNode = &pWriter->aNodeWriter[iLayer]; + int rc = SQLITE_OK; + int nPrefix; + int nSuffix; + int nSpace; + + /* Figure out how much space the key will consume if it is written to + ** the current node of layer iLayer. Due to the prefix compression, + ** the space required changes depending on which node the key is to + ** be added to. */ + nPrefix = fts3PrefixCompress(pNode->key.a, pNode->key.n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + nSpace = sqlite3Fts3VarintLen(nPrefix); + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + + if( pNode->key.n==0 || (pNode->block.n + nSpace)<=p->nNodeSize ){ + /* If the current node of layer iLayer contains zero keys, or if adding + ** the key to it will not cause it to grow to larger than nNodeSize + ** bytes in size, write the key here. */ + + Blob *pBlk = &pNode->block; + if( pBlk->n==0 ){ + blobGrowBuffer(pBlk, p->nNodeSize, &rc); + if( rc==SQLITE_OK ){ + pBlk->a[0] = (char)iLayer; + pBlk->n = 1 + sqlite3Fts3PutVarint(&pBlk->a[1], iPtr); + } + } + blobGrowBuffer(pBlk, pBlk->n + nSpace, &rc); + blobGrowBuffer(&pNode->key, nTerm, &rc); + + if( rc==SQLITE_OK ){ + if( pNode->key.n ){ + pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nPrefix); + } + pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nSuffix); + memcpy(&pBlk->a[pBlk->n], &zTerm[nPrefix], nSuffix); + pBlk->n += nSuffix; + + memcpy(pNode->key.a, zTerm, nTerm); + pNode->key.n = nTerm; + } + }else{ + /* Otherwise, flush the the current node of layer iLayer to disk. + ** Then allocate a new, empty sibling node. The key will be written + ** into the parent of this node. */ + rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n); + + assert( pNode->block.nAlloc>=p->nNodeSize ); + pNode->block.a[0] = (char)iLayer; + pNode->block.n = 1 + sqlite3Fts3PutVarint(&pNode->block.a[1], iPtr+1); + + iNextPtr = pNode->iBlock; + pNode->iBlock++; + pNode->key.n = 0; + } + + if( rc!=SQLITE_OK || iNextPtr==0 ) return rc; + iPtr = iNextPtr; + } + + assert( 0 ); + return 0; +} + +/* +** Append a term and (optionally) doclist to the FTS segment node currently +** stored in blob *pNode. The node need not contain any terms, but the +** header must be written before this function is called. +** +** A node header is a single 0x00 byte for a leaf node, or a height varint +** followed by the left-hand-child varint for an internal node. +** +** The term to be appended is passed via arguments zTerm/nTerm. For a +** leaf node, the doclist is passed as aDoclist/nDoclist. For an internal +** node, both aDoclist and nDoclist must be passed 0. +** +** If the size of the value in blob pPrev is zero, then this is the first +** term written to the node. Otherwise, pPrev contains a copy of the +** previous term. Before this function returns, it is updated to contain a +** copy of zTerm/nTerm. +** +** It is assumed that the buffer associated with pNode is already large +** enough to accommodate the new entry. The buffer associated with pPrev +** is extended by this function if requrired. +** +** If an error (i.e. OOM condition) occurs, an SQLite error code is +** returned. Otherwise, SQLITE_OK. +*/ +static int fts3AppendToNode( + Blob *pNode, /* Current node image to append to */ + Blob *pPrev, /* Buffer containing previous term written */ + const char *zTerm, /* New term to write */ + int nTerm, /* Size of zTerm in bytes */ + const char *aDoclist, /* Doclist (or NULL) to write */ + int nDoclist /* Size of aDoclist in bytes */ +){ + int rc = SQLITE_OK; /* Return code */ + int bFirst = (pPrev->n==0); /* True if this is the first term written */ + int nPrefix; /* Size of term prefix in bytes */ + int nSuffix; /* Size of term suffix in bytes */ + + /* Node must have already been started. There must be a doclist for a + ** leaf node, and there must not be a doclist for an internal node. */ + assert( pNode->n>0 ); + assert( (pNode->a[0]=='\0')==(aDoclist!=0) ); + + blobGrowBuffer(pPrev, nTerm, &rc); + if( rc!=SQLITE_OK ) return rc; + + nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + memcpy(pPrev->a, zTerm, nTerm); + pPrev->n = nTerm; + + if( bFirst==0 ){ + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nPrefix); + } + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nSuffix); + memcpy(&pNode->a[pNode->n], &zTerm[nPrefix], nSuffix); + pNode->n += nSuffix; + + if( aDoclist ){ + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nDoclist); + memcpy(&pNode->a[pNode->n], aDoclist, nDoclist); + pNode->n += nDoclist; + } + + assert( pNode->n<=pNode->nAlloc ); + + return SQLITE_OK; +} + +/* +** Append the current term and doclist pointed to by cursor pCsr to the +** appendable b-tree segment opened for writing by pWriter. +** +** Return SQLITE_OK if successful, or an SQLite error code otherwise. +*/ +static int fts3IncrmergeAppend( + Fts3Table *p, /* Fts3 table handle */ + IncrmergeWriter *pWriter, /* Writer object */ + Fts3MultiSegReader *pCsr /* Cursor containing term and doclist */ +){ + const char *zTerm = pCsr->zTerm; + int nTerm = pCsr->nTerm; + const char *aDoclist = pCsr->aDoclist; + int nDoclist = pCsr->nDoclist; + int rc = SQLITE_OK; /* Return code */ + int nSpace; /* Total space in bytes required on leaf */ + int nPrefix; /* Size of prefix shared with previous term */ + int nSuffix; /* Size of suffix (nTerm - nPrefix) */ + NodeWriter *pLeaf; /* Object used to write leaf nodes */ + + pLeaf = &pWriter->aNodeWriter[0]; + nPrefix = fts3PrefixCompress(pLeaf->key.a, pLeaf->key.n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + + nSpace = sqlite3Fts3VarintLen(nPrefix); + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; + + /* If the current block is not empty, and if adding this term/doclist + ** to the current block would make it larger than Fts3Table.nNodeSize + ** bytes, write this block out to the database. */ + if( pLeaf->block.n>0 && (pLeaf->block.n + nSpace)>p->nNodeSize ){ + rc = fts3WriteSegment(p, pLeaf->iBlock, pLeaf->block.a, pLeaf->block.n); + pWriter->nWork++; + + /* Add the current term to the parent node. The term added to the + ** parent must: + ** + ** a) be greater than the largest term on the leaf node just written + ** to the database (still available in pLeaf->key), and + ** + ** b) be less than or equal to the term about to be added to the new + ** leaf node (zTerm/nTerm). + ** + ** In other words, it must be the prefix of zTerm 1 byte longer than + ** the common prefix (if any) of zTerm and pWriter->zTerm. + */ + if( rc==SQLITE_OK ){ + rc = fts3IncrmergePush(p, pWriter, zTerm, nPrefix+1); + } + + /* Advance to the next output block */ + pLeaf->iBlock++; + pLeaf->key.n = 0; + pLeaf->block.n = 0; + + nSuffix = nTerm; + nSpace = 1; + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; + } + + blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc); + + if( rc==SQLITE_OK ){ + if( pLeaf->block.n==0 ){ + pLeaf->block.n = 1; + pLeaf->block.a[0] = '\0'; + } + rc = fts3AppendToNode( + &pLeaf->block, &pLeaf->key, zTerm, nTerm, aDoclist, nDoclist + ); + } + + return rc; +} + +/* +** This function is called to release all dynamic resources held by the +** merge-writer object pWriter, and if no error has occurred, to flush +** all outstanding node buffers held by pWriter to disk. +** +** If *pRc is not SQLITE_OK when this function is called, then no attempt +** is made to write any data to disk. Instead, this function serves only +** to release outstanding resources. +** +** Otherwise, if *pRc is initially SQLITE_OK and an error occurs while +** flushing buffers to disk, *pRc is set to an SQLite error code before +** returning. +*/ +static void fts3IncrmergeRelease( + Fts3Table *p, /* FTS3 table handle */ + IncrmergeWriter *pWriter, /* Merge-writer object */ + int *pRc /* IN/OUT: Error code */ +){ + int i; /* Used to iterate through non-root layers */ + int iRoot; /* Index of root in pWriter->aNodeWriter */ + NodeWriter *pRoot; /* NodeWriter for root node */ + int rc = *pRc; /* Error code */ + + /* Set iRoot to the index in pWriter->aNodeWriter[] of the output segment + ** root node. If the segment fits entirely on a single leaf node, iRoot + ** will be set to 0. If the root node is the parent of the leaves, iRoot + ** will be 1. And so on. */ + for(iRoot=FTS_MAX_APPENDABLE_HEIGHT-1; iRoot>=0; iRoot--){ + NodeWriter *pNode = &pWriter->aNodeWriter[iRoot]; + if( pNode->block.n>0 ) break; + assert( *pRc || pNode->block.nAlloc==0 ); + assert( *pRc || pNode->key.nAlloc==0 ); + sqlite3_free(pNode->block.a); + sqlite3_free(pNode->key.a); + } + + /* Empty output segment. This is a no-op. */ + if( iRoot<0 ) return; + + /* The entire output segment fits on a single node. Normally, this means + ** the node would be stored as a blob in the "root" column of the %_segdir + ** table. However, this is not permitted in this case. The problem is that + ** space has already been reserved in the %_segments table, and so the + ** start_block and end_block fields of the %_segdir table must be populated. + ** And, by design or by accident, released versions of FTS cannot handle + ** segments that fit entirely on the root node with start_block!=0. + ** + ** Instead, create a synthetic root node that contains nothing but a + ** pointer to the single content node. So that the segment consists of a + ** single leaf and a single interior (root) node. + ** + ** Todo: Better might be to defer allocating space in the %_segments + ** table until we are sure it is needed. + */ + if( iRoot==0 ){ + Blob *pBlock = &pWriter->aNodeWriter[1].block; + blobGrowBuffer(pBlock, 1 + FTS3_VARINT_MAX, &rc); + if( rc==SQLITE_OK ){ + pBlock->a[0] = 0x01; + pBlock->n = 1 + sqlite3Fts3PutVarint( + &pBlock->a[1], pWriter->aNodeWriter[0].iBlock + ); + } + iRoot = 1; + } + pRoot = &pWriter->aNodeWriter[iRoot]; + + /* Flush all currently outstanding nodes to disk. */ + for(i=0; i<iRoot; i++){ + NodeWriter *pNode = &pWriter->aNodeWriter[i]; + if( pNode->block.n>0 && rc==SQLITE_OK ){ + rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n); + } + sqlite3_free(pNode->block.a); + sqlite3_free(pNode->key.a); + } + + /* Write the %_segdir record. */ + if( rc==SQLITE_OK ){ + rc = fts3WriteSegdir(p, + pWriter->iAbsLevel+1, /* level */ + pWriter->iIdx, /* idx */ + pWriter->iStart, /* start_block */ + pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */ + pWriter->iEnd, /* end_block */ + pRoot->block.a, pRoot->block.n /* root */ + ); + } + sqlite3_free(pRoot->block.a); + sqlite3_free(pRoot->key.a); + + *pRc = rc; +} + +/* +** Compare the term in buffer zLhs (size in bytes nLhs) with that in +** zRhs (size in bytes nRhs) using memcmp. If one term is a prefix of +** the other, it is considered to be smaller than the other. +** +** Return -ve if zLhs is smaller than zRhs, 0 if it is equal, or +ve +** if it is greater. +*/ +static int fts3TermCmp( + const char *zLhs, int nLhs, /* LHS of comparison */ + const char *zRhs, int nRhs /* RHS of comparison */ +){ + int nCmp = MIN(nLhs, nRhs); + int res; + + res = memcmp(zLhs, zRhs, nCmp); + if( res==0 ) res = nLhs - nRhs; + + return res; +} + + +/* +** Query to see if the entry in the %_segments table with blockid iEnd is +** NULL. If no error occurs and the entry is NULL, set *pbRes 1 before +** returning. Otherwise, set *pbRes to 0. +** +** Or, if an error occurs while querying the database, return an SQLite +** error code. The final value of *pbRes is undefined in this case. +** +** This is used to test if a segment is an "appendable" segment. If it +** is, then a NULL entry has been inserted into the %_segments table +** with blockid %_segdir.end_block. +*/ +static int fts3IsAppendable(Fts3Table *p, sqlite3_int64 iEnd, int *pbRes){ + int bRes = 0; /* Result to set *pbRes to */ + sqlite3_stmt *pCheck = 0; /* Statement to query database with */ + int rc; /* Return code */ + + rc = fts3SqlStmt(p, SQL_SEGMENT_IS_APPENDABLE, &pCheck, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pCheck, 1, iEnd); + if( SQLITE_ROW==sqlite3_step(pCheck) ) bRes = 1; + rc = sqlite3_reset(pCheck); + } + + *pbRes = bRes; + return rc; +} + +/* +** This function is called when initializing an incremental-merge operation. +** It checks if the existing segment with index value iIdx at absolute level +** (iAbsLevel+1) can be appended to by the incremental merge. If it can, the +** merge-writer object *pWriter is initialized to write to it. +** +** An existing segment can be appended to by an incremental merge if: +** +** * It was initially created as an appendable segment (with all required +** space pre-allocated), and +** +** * The first key read from the input (arguments zKey and nKey) is +** greater than the largest key currently stored in the potential +** output segment. +*/ +static int fts3IncrmergeLoad( + Fts3Table *p, /* Fts3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ + int iIdx, /* Index of candidate output segment */ + const char *zKey, /* First key to write */ + int nKey, /* Number of bytes in nKey */ + IncrmergeWriter *pWriter /* Populate this object */ +){ + int rc; /* Return code */ + sqlite3_stmt *pSelect = 0; /* SELECT to read %_segdir entry */ + + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pSelect, 0); + if( rc==SQLITE_OK ){ + sqlite3_int64 iStart = 0; /* Value of %_segdir.start_block */ + sqlite3_int64 iLeafEnd = 0; /* Value of %_segdir.leaves_end_block */ + sqlite3_int64 iEnd = 0; /* Value of %_segdir.end_block */ + const char *aRoot = 0; /* Pointer to %_segdir.root buffer */ + int nRoot = 0; /* Size of aRoot[] in bytes */ + int rc2; /* Return code from sqlite3_reset() */ + int bAppendable = 0; /* Set to true if segment is appendable */ + + /* Read the %_segdir entry for index iIdx absolute level (iAbsLevel+1) */ + sqlite3_bind_int64(pSelect, 1, iAbsLevel+1); + sqlite3_bind_int(pSelect, 2, iIdx); + if( sqlite3_step(pSelect)==SQLITE_ROW ){ + iStart = sqlite3_column_int64(pSelect, 1); + iLeafEnd = sqlite3_column_int64(pSelect, 2); + iEnd = sqlite3_column_int64(pSelect, 3); + nRoot = sqlite3_column_bytes(pSelect, 4); + aRoot = sqlite3_column_blob(pSelect, 4); + }else{ + return sqlite3_reset(pSelect); + } + + /* Check for the zero-length marker in the %_segments table */ + rc = fts3IsAppendable(p, iEnd, &bAppendable); + + /* Check that zKey/nKey is larger than the largest key the candidate */ + if( rc==SQLITE_OK && bAppendable ){ + char *aLeaf = 0; + int nLeaf = 0; + + rc = sqlite3Fts3ReadBlock(p, iLeafEnd, &aLeaf, &nLeaf, 0); + if( rc==SQLITE_OK ){ + NodeReader reader; + for(rc = nodeReaderInit(&reader, aLeaf, nLeaf); + rc==SQLITE_OK && reader.aNode; + rc = nodeReaderNext(&reader) + ){ + assert( reader.aNode ); + } + if( fts3TermCmp(zKey, nKey, reader.term.a, reader.term.n)<=0 ){ + bAppendable = 0; + } + nodeReaderRelease(&reader); + } + sqlite3_free(aLeaf); + } + + if( rc==SQLITE_OK && bAppendable ){ + /* It is possible to append to this segment. Set up the IncrmergeWriter + ** object to do so. */ + int i; + int nHeight = (int)aRoot[0]; + NodeWriter *pNode; + + pWriter->nLeafEst = (int)((iEnd - iStart) + 1)/FTS_MAX_APPENDABLE_HEIGHT; + pWriter->iStart = iStart; + pWriter->iEnd = iEnd; + pWriter->iAbsLevel = iAbsLevel; + pWriter->iIdx = iIdx; + + for(i=nHeight+1; i<FTS_MAX_APPENDABLE_HEIGHT; i++){ + pWriter->aNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; + } + + pNode = &pWriter->aNodeWriter[nHeight]; + pNode->iBlock = pWriter->iStart + pWriter->nLeafEst*nHeight; + blobGrowBuffer(&pNode->block, MAX(nRoot, p->nNodeSize), &rc); + if( rc==SQLITE_OK ){ + memcpy(pNode->block.a, aRoot, nRoot); + pNode->block.n = nRoot; + } + + for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){ + NodeReader reader; + pNode = &pWriter->aNodeWriter[i]; + + rc = nodeReaderInit(&reader, pNode->block.a, pNode->block.n); + while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader); + blobGrowBuffer(&pNode->key, reader.term.n, &rc); + if( rc==SQLITE_OK ){ + memcpy(pNode->key.a, reader.term.a, reader.term.n); + pNode->key.n = reader.term.n; + if( i>0 ){ + char *aBlock = 0; + int nBlock = 0; + pNode = &pWriter->aNodeWriter[i-1]; + pNode->iBlock = reader.iChild; + rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock, 0); + blobGrowBuffer(&pNode->block, MAX(nBlock, p->nNodeSize), &rc); + if( rc==SQLITE_OK ){ + memcpy(pNode->block.a, aBlock, nBlock); + pNode->block.n = nBlock; + } + sqlite3_free(aBlock); + } + } + nodeReaderRelease(&reader); + } + } + + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; + } + + return rc; +} + +/* +** Determine the largest segment index value that exists within absolute +** level iAbsLevel+1. If no error occurs, set *piIdx to this value plus +** one before returning SQLITE_OK. Or, if there are no segments at all +** within level iAbsLevel, set *piIdx to zero. +** +** If an error occurs, return an SQLite error code. The final value of +** *piIdx is undefined in this case. +*/ +static int fts3IncrmergeOutputIdx( + Fts3Table *p, /* FTS Table handle */ + sqlite3_int64 iAbsLevel, /* Absolute index of input segments */ + int *piIdx /* OUT: Next free index at iAbsLevel+1 */ +){ + int rc; + sqlite3_stmt *pOutputIdx = 0; /* SQL used to find output index */ + + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pOutputIdx, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pOutputIdx, 1, iAbsLevel+1); + sqlite3_step(pOutputIdx); + *piIdx = sqlite3_column_int(pOutputIdx, 0); + rc = sqlite3_reset(pOutputIdx); + } + + return rc; +} + +/* +** Allocate an appendable output segment on absolute level iAbsLevel+1 +** with idx value iIdx. +** +** In the %_segdir table, a segment is defined by the values in three +** columns: +** +** start_block +** leaves_end_block +** end_block +** +** When an appendable segment is allocated, it is estimated that the +** maximum number of leaf blocks that may be required is the sum of the +** number of leaf blocks consumed by the input segments, plus the number +** of input segments, multiplied by two. This value is stored in stack +** variable nLeafEst. +** +** A total of 16*nLeafEst blocks are allocated when an appendable segment +** is created ((1 + end_block - start_block)==16*nLeafEst). The contiguous +** array of leaf nodes starts at the first block allocated. The array +** of interior nodes that are parents of the leaf nodes start at block +** (start_block + (1 + end_block - start_block) / 16). And so on. +** +** In the actual code below, the value "16" is replaced with the +** pre-processor macro FTS_MAX_APPENDABLE_HEIGHT. +*/ +static int fts3IncrmergeWriter( + Fts3Table *p, /* Fts3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ + int iIdx, /* Index of new output segment */ + Fts3MultiSegReader *pCsr, /* Cursor that data will be read from */ + IncrmergeWriter *pWriter /* Populate this object */ +){ + int rc; /* Return Code */ + int i; /* Iterator variable */ + int nLeafEst = 0; /* Blocks allocated for leaf nodes */ + sqlite3_stmt *pLeafEst = 0; /* SQL used to determine nLeafEst */ + sqlite3_stmt *pFirstBlock = 0; /* SQL used to determine first block */ + + /* Calculate nLeafEst. */ + rc = fts3SqlStmt(p, SQL_MAX_LEAF_NODE_ESTIMATE, &pLeafEst, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pLeafEst, 1, iAbsLevel); + sqlite3_bind_int64(pLeafEst, 2, pCsr->nSegment); + if( SQLITE_ROW==sqlite3_step(pLeafEst) ){ + nLeafEst = sqlite3_column_int(pLeafEst, 0); + } + rc = sqlite3_reset(pLeafEst); + } + if( rc!=SQLITE_OK ) return rc; + + /* Calculate the first block to use in the output segment */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pFirstBlock, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pFirstBlock) ){ + pWriter->iStart = sqlite3_column_int64(pFirstBlock, 0); + pWriter->iEnd = pWriter->iStart - 1; + pWriter->iEnd += nLeafEst * FTS_MAX_APPENDABLE_HEIGHT; + } + rc = sqlite3_reset(pFirstBlock); + } + if( rc!=SQLITE_OK ) return rc; + + /* Insert the marker in the %_segments table to make sure nobody tries + ** to steal the space just allocated. This is also used to identify + ** appendable segments. */ + rc = fts3WriteSegment(p, pWriter->iEnd, 0, 0); + if( rc!=SQLITE_OK ) return rc; + + pWriter->iAbsLevel = iAbsLevel; + pWriter->nLeafEst = nLeafEst; + pWriter->iIdx = iIdx; + + /* Set up the array of NodeWriter objects */ + for(i=0; i<FTS_MAX_APPENDABLE_HEIGHT; i++){ + pWriter->aNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; + } + return SQLITE_OK; +} + +/* +** Remove an entry from the %_segdir table. This involves running the +** following two statements: +** +** DELETE FROM %_segdir WHERE level = :iAbsLevel AND idx = :iIdx +** UPDATE %_segdir SET idx = idx - 1 WHERE level = :iAbsLevel AND idx > :iIdx +** +** The DELETE statement removes the specific %_segdir level. The UPDATE +** statement ensures that the remaining segments have contiguously allocated +** idx values. +*/ +static int fts3RemoveSegdirEntry( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level to delete from */ + int iIdx /* Index of %_segdir entry to delete */ +){ + int rc; /* Return code */ + sqlite3_stmt *pDelete = 0; /* DELETE statement */ + + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_ENTRY, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDelete, 1, iAbsLevel); + sqlite3_bind_int(pDelete, 2, iIdx); + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); + } + + return rc; +} + +/* +** One or more segments have just been removed from absolute level iAbsLevel. +** Update the 'idx' values of the remaining segments in the level so that +** the idx values are a contiguous sequence starting from 0. +*/ +static int fts3RepackSegdirLevel( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel /* Absolute level to repack */ +){ + int rc; /* Return code */ + int *aIdx = 0; /* Array of remaining idx values */ + int nIdx = 0; /* Valid entries in aIdx[] */ + int nAlloc = 0; /* Allocated size of aIdx[] */ + int i; /* Iterator variable */ + sqlite3_stmt *pSelect = 0; /* Select statement to read idx values */ + sqlite3_stmt *pUpdate = 0; /* Update statement to modify idx values */ + + rc = fts3SqlStmt(p, SQL_SELECT_INDEXES, &pSelect, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int64(pSelect, 1, iAbsLevel); + while( SQLITE_ROW==sqlite3_step(pSelect) ){ + if( nIdx>=nAlloc ){ + int *aNew; + nAlloc += 16; + aNew = sqlite3_realloc(aIdx, nAlloc*sizeof(int)); + if( !aNew ){ + rc = SQLITE_NOMEM; + break; + } + aIdx = aNew; + } + aIdx[nIdx++] = sqlite3_column_int(pSelect, 0); + } + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; + } + + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_SHIFT_SEGDIR_ENTRY, &pUpdate, 0); + } + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pUpdate, 2, iAbsLevel); + } + + assert( p->bIgnoreSavepoint==0 ); + p->bIgnoreSavepoint = 1; + for(i=0; rc==SQLITE_OK && i<nIdx; i++){ + if( aIdx[i]!=i ){ + sqlite3_bind_int(pUpdate, 3, aIdx[i]); + sqlite3_bind_int(pUpdate, 1, i); + sqlite3_step(pUpdate); + rc = sqlite3_reset(pUpdate); + } + } + p->bIgnoreSavepoint = 0; + + sqlite3_free(aIdx); + return rc; +} + +static void fts3StartNode(Blob *pNode, int iHeight, sqlite3_int64 iChild){ + pNode->a[0] = (char)iHeight; + if( iChild ){ + assert( pNode->nAlloc>=1+sqlite3Fts3VarintLen(iChild) ); + pNode->n = 1 + sqlite3Fts3PutVarint(&pNode->a[1], iChild); + }else{ + assert( pNode->nAlloc>=1 ); + pNode->n = 1; + } +} + +/* +** The first two arguments are a pointer to and the size of a segment b-tree +** node. The node may be a leaf or an internal node. +** +** This function creates a new node image in blob object *pNew by copying +** all terms that are greater than or equal to zTerm/nTerm (for leaf nodes) +** or greater than zTerm/nTerm (for internal nodes) from aNode/nNode. +*/ +static int fts3TruncateNode( + const char *aNode, /* Current node image */ + int nNode, /* Size of aNode in bytes */ + Blob *pNew, /* OUT: Write new node image here */ + const char *zTerm, /* Omit all terms smaller than this */ + int nTerm, /* Size of zTerm in bytes */ + sqlite3_int64 *piBlock /* OUT: Block number in next layer down */ +){ + NodeReader reader; /* Reader object */ + Blob prev = {0, 0, 0}; /* Previous term written to new node */ + int rc = SQLITE_OK; /* Return code */ + int bLeaf = aNode[0]=='\0'; /* True for a leaf node */ + + /* Allocate required output space */ + blobGrowBuffer(pNew, nNode, &rc); + if( rc!=SQLITE_OK ) return rc; + pNew->n = 0; + + /* Populate new node buffer */ + for(rc = nodeReaderInit(&reader, aNode, nNode); + rc==SQLITE_OK && reader.aNode; + rc = nodeReaderNext(&reader) + ){ + if( pNew->n==0 ){ + int res = fts3TermCmp(reader.term.a, reader.term.n, zTerm, nTerm); + if( res<0 || (bLeaf==0 && res==0) ) continue; + fts3StartNode(pNew, (int)aNode[0], reader.iChild); + *piBlock = reader.iChild; + } + rc = fts3AppendToNode( + pNew, &prev, reader.term.a, reader.term.n, + reader.aDoclist, reader.nDoclist + ); + if( rc!=SQLITE_OK ) break; + } + if( pNew->n==0 ){ + fts3StartNode(pNew, (int)aNode[0], reader.iChild); + *piBlock = reader.iChild; + } + assert( pNew->n<=pNew->nAlloc ); + + nodeReaderRelease(&reader); + sqlite3_free(prev.a); + return rc; +} + +/* +** Remove all terms smaller than zTerm/nTerm from segment iIdx in absolute +** level iAbsLevel. This may involve deleting entries from the %_segments +** table, and modifying existing entries in both the %_segments and %_segdir +** tables. +** +** SQLITE_OK is returned if the segment is updated successfully. Or an +** SQLite error code otherwise. +*/ +static int fts3TruncateSegment( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of segment to modify */ + int iIdx, /* Index within level of segment to modify */ + const char *zTerm, /* Remove terms smaller than this */ + int nTerm /* Number of bytes in buffer zTerm */ +){ + int rc = SQLITE_OK; /* Return code */ + Blob root = {0,0,0}; /* New root page image */ + Blob block = {0,0,0}; /* Buffer used for any other block */ + sqlite3_int64 iBlock = 0; /* Block id */ + sqlite3_int64 iNewStart = 0; /* New value for iStartBlock */ + sqlite3_int64 iOldStart = 0; /* Old value for iStartBlock */ + sqlite3_stmt *pFetch = 0; /* Statement used to fetch segdir */ + + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pFetch, 0); + if( rc==SQLITE_OK ){ + int rc2; /* sqlite3_reset() return code */ + sqlite3_bind_int64(pFetch, 1, iAbsLevel); + sqlite3_bind_int(pFetch, 2, iIdx); + if( SQLITE_ROW==sqlite3_step(pFetch) ){ + const char *aRoot = sqlite3_column_blob(pFetch, 4); + int nRoot = sqlite3_column_bytes(pFetch, 4); + iOldStart = sqlite3_column_int64(pFetch, 1); + rc = fts3TruncateNode(aRoot, nRoot, &root, zTerm, nTerm, &iBlock); + } + rc2 = sqlite3_reset(pFetch); + if( rc==SQLITE_OK ) rc = rc2; + } + + while( rc==SQLITE_OK && iBlock ){ + char *aBlock = 0; + int nBlock = 0; + iNewStart = iBlock; + + rc = sqlite3Fts3ReadBlock(p, iBlock, &aBlock, &nBlock, 0); + if( rc==SQLITE_OK ){ + rc = fts3TruncateNode(aBlock, nBlock, &block, zTerm, nTerm, &iBlock); + } + if( rc==SQLITE_OK ){ + rc = fts3WriteSegment(p, iNewStart, block.a, block.n); + } + sqlite3_free(aBlock); + } + + /* Variable iNewStart now contains the first valid leaf node. */ + if( rc==SQLITE_OK && iNewStart ){ + sqlite3_stmt *pDel = 0; + rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDel, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDel, 1, iOldStart); + sqlite3_bind_int64(pDel, 2, iNewStart-1); + sqlite3_step(pDel); + rc = sqlite3_reset(pDel); + } + } + + if( rc==SQLITE_OK ){ + sqlite3_stmt *pChomp = 0; + rc = fts3SqlStmt(p, SQL_CHOMP_SEGDIR, &pChomp, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pChomp, 1, iNewStart); + sqlite3_bind_blob(pChomp, 2, root.a, root.n, SQLITE_STATIC); + sqlite3_bind_int64(pChomp, 3, iAbsLevel); + sqlite3_bind_int(pChomp, 4, iIdx); + sqlite3_step(pChomp); + rc = sqlite3_reset(pChomp); + } + } + + sqlite3_free(root.a); + sqlite3_free(block.a); + return rc; +} + +/* +** This function is called after an incrmental-merge operation has run to +** merge (or partially merge) two or more segments from absolute level +** iAbsLevel. +** +** Each input segment is either removed from the db completely (if all of +** its data was copied to the output segment by the incrmerge operation) +** or modified in place so that it no longer contains those entries that +** have been duplicated in the output segment. +*/ +static int fts3IncrmergeChomp( + Fts3Table *p, /* FTS table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level containing segments */ + Fts3MultiSegReader *pCsr, /* Chomp all segments opened by this cursor */ + int *pnRem /* Number of segments not deleted */ +){ + int i; + int nRem = 0; + int rc = SQLITE_OK; + + for(i=pCsr->nSegment-1; i>=0 && rc==SQLITE_OK; i--){ + Fts3SegReader *pSeg = 0; + int j; + + /* Find the Fts3SegReader object with Fts3SegReader.iIdx==i. It is hiding + ** somewhere in the pCsr->apSegment[] array. */ + for(j=0; ALWAYS(j<pCsr->nSegment); j++){ + pSeg = pCsr->apSegment[j]; + if( pSeg->iIdx==i ) break; + } + assert( j<pCsr->nSegment && pSeg->iIdx==i ); + + if( pSeg->aNode==0 ){ + /* Seg-reader is at EOF. Remove the entire input segment. */ + rc = fts3DeleteSegment(p, pSeg); + if( rc==SQLITE_OK ){ + rc = fts3RemoveSegdirEntry(p, iAbsLevel, pSeg->iIdx); + } + *pnRem = 0; + }else{ + /* The incremental merge did not copy all the data from this + ** segment to the upper level. The segment is modified in place + ** so that it contains no keys smaller than zTerm/nTerm. */ + const char *zTerm = pSeg->zTerm; + int nTerm = pSeg->nTerm; + rc = fts3TruncateSegment(p, iAbsLevel, pSeg->iIdx, zTerm, nTerm); + nRem++; + } + } + + if( rc==SQLITE_OK && nRem!=pCsr->nSegment ){ + rc = fts3RepackSegdirLevel(p, iAbsLevel); + } + + *pnRem = nRem; + return rc; +} + +/* +** Store an incr-merge hint in the database. +*/ +static int fts3IncrmergeHintStore(Fts3Table *p, Blob *pHint){ + sqlite3_stmt *pReplace = 0; + int rc; /* Return code */ + + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pReplace, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pReplace, 1, FTS_STAT_INCRMERGEHINT); + sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC); + sqlite3_step(pReplace); + rc = sqlite3_reset(pReplace); + } + + return rc; +} + +/* +** Load an incr-merge hint from the database. The incr-merge hint, if one +** exists, is stored in the rowid==1 row of the %_stat table. +** +** If successful, populate blob *pHint with the value read from the %_stat +** table and return SQLITE_OK. Otherwise, if an error occurs, return an +** SQLite error code. +*/ +static int fts3IncrmergeHintLoad(Fts3Table *p, Blob *pHint){ + sqlite3_stmt *pSelect = 0; + int rc; + + pHint->n = 0; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pSelect, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int(pSelect, 1, FTS_STAT_INCRMERGEHINT); + if( SQLITE_ROW==sqlite3_step(pSelect) ){ + const char *aHint = sqlite3_column_blob(pSelect, 0); + int nHint = sqlite3_column_bytes(pSelect, 0); + if( aHint ){ + blobGrowBuffer(pHint, nHint, &rc); + if( rc==SQLITE_OK ){ + memcpy(pHint->a, aHint, nHint); + pHint->n = nHint; + } + } + } + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; + } + + return rc; +} + +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, append an entry to the hint stored in blob *pHint. Each entry +** consists of two varints, the absolute level number of the input segments +** and the number of input segments. +** +** If successful, leave *pRc set to SQLITE_OK and return. If an error occurs, +** set *pRc to an SQLite error code before returning. +*/ +static void fts3IncrmergeHintPush( + Blob *pHint, /* Hint blob to append to */ + i64 iAbsLevel, /* First varint to store in hint */ + int nInput, /* Second varint to store in hint */ + int *pRc /* IN/OUT: Error code */ +){ + blobGrowBuffer(pHint, pHint->n + 2*FTS3_VARINT_MAX, pRc); + if( *pRc==SQLITE_OK ){ + pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], iAbsLevel); + pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], (i64)nInput); + } +} + +/* +** Read the last entry (most recently pushed) from the hint blob *pHint +** and then remove the entry. Write the two values read to *piAbsLevel and +** *pnInput before returning. +** +** If no error occurs, return SQLITE_OK. If the hint blob in *pHint does +** not contain at least two valid varints, return SQLITE_CORRUPT_VTAB. +*/ +static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){ + const int nHint = pHint->n; + int i; + + i = pHint->n-2; + while( i>0 && (pHint->a[i-1] & 0x80) ) i--; + while( i>0 && (pHint->a[i-1] & 0x80) ) i--; + + pHint->n = i; + i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel); + i += sqlite3Fts3GetVarint32(&pHint->a[i], pnInput); + if( i!=nHint ) return SQLITE_CORRUPT_VTAB; + + return SQLITE_OK; +} + + +/* +** Attempt an incremental merge that writes nMerge leaf blocks. +** +** Incremental merges happen nMin segments at a time. The two +** segments to be merged are the nMin oldest segments (the ones with +** the smallest indexes) in the highest level that contains at least +** nMin segments. Multiple merges might occur in an attempt to write the +** quota of nMerge leaf blocks. +*/ +int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ + int rc; /* Return code */ + int nRem = nMerge; /* Number of leaf pages yet to be written */ + Fts3MultiSegReader *pCsr; /* Cursor used to read input data */ + Fts3SegFilter *pFilter; /* Filter used with cursor pCsr */ + IncrmergeWriter *pWriter; /* Writer object */ + int nSeg = 0; /* Number of input segments */ + sqlite3_int64 iAbsLevel = 0; /* Absolute level number to work on */ + Blob hint = {0, 0, 0}; /* Hint read from %_stat table */ + int bDirtyHint = 0; /* True if blob 'hint' has been modified */ + + /* Allocate space for the cursor, filter and writer objects */ + const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter); + pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc); + if( !pWriter ) return SQLITE_NOMEM; + pFilter = (Fts3SegFilter *)&pWriter[1]; + pCsr = (Fts3MultiSegReader *)&pFilter[1]; + + rc = fts3IncrmergeHintLoad(p, &hint); + while( rc==SQLITE_OK && nRem>0 ){ + const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex; + sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ + int bUseHint = 0; /* True if attempting to append */ + + /* Search the %_segdir table for the absolute level with the smallest + ** relative level number that contains at least nMin segments, if any. + ** If one is found, set iAbsLevel to the absolute level number and + ** nSeg to nMin. If no level with at least nMin segments can be found, + ** set nSeg to -1. + */ + rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); + sqlite3_bind_int(pFindLevel, 1, nMin); + if( sqlite3_step(pFindLevel)==SQLITE_ROW ){ + iAbsLevel = sqlite3_column_int64(pFindLevel, 0); + nSeg = nMin; + }else{ + nSeg = -1; + } + rc = sqlite3_reset(pFindLevel); + + /* If the hint read from the %_stat table is not empty, check if the + ** last entry in it specifies a relative level smaller than or equal + ** to the level identified by the block above (if any). If so, this + ** iteration of the loop will work on merging at the hinted level. + */ + if( rc==SQLITE_OK && hint.n ){ + int nHint = hint.n; + sqlite3_int64 iHintAbsLevel = 0; /* Hint level */ + int nHintSeg = 0; /* Hint number of segments */ + + rc = fts3IncrmergeHintPop(&hint, &iHintAbsLevel, &nHintSeg); + if( nSeg<0 || (iAbsLevel % nMod) >= (iHintAbsLevel % nMod) ){ + iAbsLevel = iHintAbsLevel; + nSeg = nHintSeg; + bUseHint = 1; + bDirtyHint = 1; + }else{ + /* This undoes the effect of the HintPop() above - so that no entry + ** is removed from the hint blob. */ + hint.n = nHint; + } + } + + /* If nSeg is less that zero, then there is no level with at least + ** nMin segments and no hint in the %_stat table. No work to do. + ** Exit early in this case. */ + if( nSeg<0 ) break; + + /* Open a cursor to iterate through the contents of the oldest nSeg + ** indexes of absolute level iAbsLevel. If this cursor is opened using + ** the 'hint' parameters, it is possible that there are less than nSeg + ** segments available in level iAbsLevel. In this case, no work is + ** done on iAbsLevel - fall through to the next iteration of the loop + ** to start work on some other level. */ + memset(pWriter, 0, nAlloc); + pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; + if( rc==SQLITE_OK ){ + rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); + } + if( SQLITE_OK==rc && pCsr->nSegment==nSeg + && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) + && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) + ){ + int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ + rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); + if( rc==SQLITE_OK ){ + if( bUseHint && iIdx>0 ){ + const char *zKey = pCsr->zTerm; + int nKey = pCsr->nTerm; + rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter); + }else{ + rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter); + } + } + + if( rc==SQLITE_OK && pWriter->nLeafEst ){ + fts3LogMerge(nSeg, iAbsLevel); + do { + rc = fts3IncrmergeAppend(p, pWriter, pCsr); + if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr); + if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK; + }while( rc==SQLITE_ROW ); + + /* Update or delete the input segments */ + if( rc==SQLITE_OK ){ + nRem -= (1 + pWriter->nWork); + rc = fts3IncrmergeChomp(p, iAbsLevel, pCsr, &nSeg); + if( nSeg!=0 ){ + bDirtyHint = 1; + fts3IncrmergeHintPush(&hint, iAbsLevel, nSeg, &rc); + } + } + } + + fts3IncrmergeRelease(p, pWriter, &rc); + } + + sqlite3Fts3SegReaderFinish(pCsr); + } + + /* Write the hint values into the %_stat table for the next incr-merger */ + if( bDirtyHint && rc==SQLITE_OK ){ + rc = fts3IncrmergeHintStore(p, &hint); + } + + sqlite3_free(pWriter); + sqlite3_free(hint.a); + return rc; +} + +/* +** Convert the text beginning at *pz into an integer and return +** its value. Advance *pz to point to the first character past +** the integer. +*/ +static int fts3Getint(const char **pz){ + const char *z = *pz; + int i = 0; + while( (*z)>='0' && (*z)<='9' ) i = 10*i + *(z++) - '0'; + *pz = z; + return i; +} + +/* +** Process statements of the form: +** +** INSERT INTO table(table) VALUES('merge=A,B'); +** +** A and B are integers that decode to be the number of leaf pages +** written for the merge, and the minimum number of segments on a level +** before it will be selected for a merge, respectively. +*/ +static int fts3DoIncrmerge( + Fts3Table *p, /* FTS3 table handle */ + const char *zParam /* Nul-terminated string containing "A,B" */ +){ + int rc; + int nMin = (FTS3_MERGE_COUNT / 2); + int nMerge = 0; + const char *z = zParam; + + /* Read the first integer value */ + nMerge = fts3Getint(&z); + + /* If the first integer value is followed by a ',', read the second + ** integer value. */ + if( z[0]==',' && z[1]!='\0' ){ + z++; + nMin = fts3Getint(&z); + } + + if( z[0]!='\0' || nMin<2 ){ + rc = SQLITE_ERROR; + }else{ + rc = SQLITE_OK; + if( !p->bHasStat ){ + assert( p->bFts4==0 ); + sqlite3Fts3CreateStatTable(&rc, p); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3Incrmerge(p, nMerge, nMin); + } + sqlite3Fts3SegmentsClose(p); + } + return rc; +} + +/* +** Process statements of the form: +** +** INSERT INTO table(table) VALUES('automerge=X'); +** +** where X is an integer. X==0 means to turn automerge off. X!=0 means +** turn it on. The setting is persistent. +*/ +static int fts3DoAutoincrmerge( + Fts3Table *p, /* FTS3 table handle */ + const char *zParam /* Nul-terminated string containing boolean */ +){ + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = 0; + p->bAutoincrmerge = fts3Getint(&zParam)!=0; + if( !p->bHasStat ){ + assert( p->bFts4==0 ); + sqlite3Fts3CreateStatTable(&rc, p); + if( rc ) return rc; + } + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); + if( rc ) return rc;; + sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); + sqlite3_bind_int(pStmt, 2, p->bAutoincrmerge); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + return rc; +} + +/* +** Return a 64-bit checksum for the FTS index entry specified by the +** arguments to this function. +*/ +static u64 fts3ChecksumEntry( + const char *zTerm, /* Pointer to buffer containing term */ + int nTerm, /* Size of zTerm in bytes */ + int iLangid, /* Language id for current row */ + int iIndex, /* Index (0..Fts3Table.nIndex-1) */ + i64 iDocid, /* Docid for current row. */ + int iCol, /* Column number */ + int iPos /* Position */ +){ + int i; + u64 ret = (u64)iDocid; + + ret += (ret<<3) + iLangid; + ret += (ret<<3) + iIndex; + ret += (ret<<3) + iCol; + ret += (ret<<3) + iPos; + for(i=0; i<nTerm; i++) ret += (ret<<3) + zTerm[i]; + + return ret; +} + +/* +** Return a checksum of all entries in the FTS index that correspond to +** language id iLangid. The checksum is calculated by XORing the checksums +** of each individual entry (see fts3ChecksumEntry()) together. +** +** If successful, the checksum value is returned and *pRc set to SQLITE_OK. +** Otherwise, if an error occurs, *pRc is set to an SQLite error code. The +** return value is undefined in this case. +*/ +static u64 fts3ChecksumIndex( + Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language id to return cksum for */ + int iIndex, /* Index to cksum (0..p->nIndex-1) */ + int *pRc /* OUT: Return code */ +){ + Fts3SegFilter filter; + Fts3MultiSegReader csr; + int rc; + u64 cksum = 0; + + assert( *pRc==SQLITE_OK ); + + memset(&filter, 0, sizeof(filter)); + memset(&csr, 0, sizeof(csr)); + filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; + filter.flags |= FTS3_SEGMENT_SCAN; + + rc = sqlite3Fts3SegReaderCursor( + p, iLangid, iIndex, FTS3_SEGCURSOR_ALL, 0, 0, 0, 1,&csr + ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); + } + + if( rc==SQLITE_OK ){ + while( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, &csr)) ){ + char *pCsr = csr.aDoclist; + char *pEnd = &pCsr[csr.nDoclist]; + + i64 iDocid = 0; + i64 iCol = 0; + i64 iPos = 0; + + pCsr += sqlite3Fts3GetVarint(pCsr, &iDocid); + while( pCsr<pEnd ){ + i64 iVal = 0; + pCsr += sqlite3Fts3GetVarint(pCsr, &iVal); + if( pCsr<pEnd ){ + if( iVal==0 || iVal==1 ){ + iCol = 0; + iPos = 0; + if( iVal ){ + pCsr += sqlite3Fts3GetVarint(pCsr, &iCol); + }else{ + pCsr += sqlite3Fts3GetVarint(pCsr, &iVal); + iDocid += iVal; + } + }else{ + iPos += (iVal - 2); + cksum = cksum ^ fts3ChecksumEntry( + csr.zTerm, csr.nTerm, iLangid, iIndex, iDocid, + (int)iCol, (int)iPos + ); + } + } + } + } + } + sqlite3Fts3SegReaderFinish(&csr); + + *pRc = rc; + return cksum; +} + +/* +** Check if the contents of the FTS index match the current contents of the +** content table. If no error occurs and the contents do match, set *pbOk +** to true and return SQLITE_OK. Or if the contents do not match, set *pbOk +** to false before returning. +** +** If an error occurs (e.g. an OOM or IO error), return an SQLite error +** code. The final value of *pbOk is undefined in this case. +*/ +static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ + int rc = SQLITE_OK; /* Return code */ + u64 cksum1 = 0; /* Checksum based on FTS index contents */ + u64 cksum2 = 0; /* Checksum based on %_content contents */ + sqlite3_stmt *pAllLangid = 0; /* Statement to return all language-ids */ + + /* This block calculates the checksum according to the FTS index. */ + rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int(pAllLangid, 1, p->nIndex); + while( rc==SQLITE_OK && sqlite3_step(pAllLangid)==SQLITE_ROW ){ + int iLangid = sqlite3_column_int(pAllLangid, 0); + int i; + for(i=0; i<p->nIndex; i++){ + cksum1 = cksum1 ^ fts3ChecksumIndex(p, iLangid, i, &rc); + } + } + rc2 = sqlite3_reset(pAllLangid); + if( rc==SQLITE_OK ) rc = rc2; + } + + /* This block calculates the checksum according to the %_content table */ + rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); + if( rc==SQLITE_OK ){ + sqlite3_tokenizer_module const *pModule = p->pTokenizer->pModule; + sqlite3_stmt *pStmt = 0; + char *zSql; + + zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + i64 iDocid = sqlite3_column_int64(pStmt, 0); + int iLang = langidFromSelect(p, pStmt); + int iCol; + + for(iCol=0; rc==SQLITE_OK && iCol<p->nColumn; iCol++){ + const char *zText = (const char *)sqlite3_column_text(pStmt, iCol+1); + int nText = sqlite3_column_bytes(pStmt, iCol+1); + sqlite3_tokenizer_cursor *pT = 0; + + rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, nText, &pT); + while( rc==SQLITE_OK ){ + char const *zToken; /* Buffer containing token */ + int nToken; /* Number of bytes in token */ + int iDum1, iDum2; /* Dummy variables */ + int iPos; /* Position of token in zText */ + + rc = pModule->xNext(pT, &zToken, &nToken, &iDum1, &iDum2, &iPos); + if( rc==SQLITE_OK ){ + int i; + cksum2 = cksum2 ^ fts3ChecksumEntry( + zToken, nToken, iLang, 0, iDocid, iCol, iPos + ); + for(i=1; i<p->nIndex; i++){ + if( p->aIndex[i].nPrefix<=nToken ){ + cksum2 = cksum2 ^ fts3ChecksumEntry( + zToken, p->aIndex[i].nPrefix, iLang, i, iDocid, iCol, iPos + ); + } + } + } + } + if( pT ) pModule->xClose(pT); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + } + + sqlite3_finalize(pStmt); + } + + *pbOk = (cksum1==cksum2); + return rc; +} + +/* +** Run the integrity-check. If no error occurs and the current contents of +** the FTS index are correct, return SQLITE_OK. Or, if the contents of the +** FTS index are incorrect, return SQLITE_CORRUPT_VTAB. +** +** Or, if an error (e.g. an OOM or IO error) occurs, return an SQLite +** error code. +** +** The integrity-check works as follows. For each token and indexed token +** prefix in the document set, a 64-bit checksum is calculated (by code +** in fts3ChecksumEntry()) based on the following: +** +** + The index number (0 for the main index, 1 for the first prefix +** index etc.), +** + The token (or token prefix) text itself, +** + The language-id of the row it appears in, +** + The docid of the row it appears in, +** + The column it appears in, and +** + The tokens position within that column. +** +** The checksums for all entries in the index are XORed together to create +** a single checksum for the entire index. +** +** The integrity-check code calculates the same checksum in two ways: +** +** 1. By scanning the contents of the FTS index, and +** 2. By scanning and tokenizing the content table. +** +** If the two checksums are identical, the integrity-check is deemed to have +** passed. +*/ +static int fts3DoIntegrityCheck( + Fts3Table *p /* FTS3 table handle */ +){ + int rc; + int bOk = 0; + rc = fts3IntegrityCheck(p, &bOk); + if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_CORRUPT_VTAB; + return rc; +} + /* ** Handle a 'special' INSERT of the form: ** @@ -3189,6 +5018,12 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ rc = fts3DoOptimize(p, 0); }else if( nVal==7 && 0==sqlite3_strnicmp(zVal, "rebuild", 7) ){ rc = fts3DoRebuild(p); + }else if( nVal==15 && 0==sqlite3_strnicmp(zVal, "integrity-check", 15) ){ + rc = fts3DoIntegrityCheck(p); + }else if( nVal>6 && 0==sqlite3_strnicmp(zVal, "merge=", 6) ){ + rc = fts3DoIncrmerge(p, &zVal[6]); + }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){ + rc = fts3DoAutoincrmerge(p, &zVal[10]); #ifdef SQLITE_TEST }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ p->nNodeSize = atoi(&zVal[9]); @@ -3384,7 +5219,7 @@ static int fts3DeleteByRowid( ** tables. The schema of the virtual table being: ** ** CREATE TABLE <table name>( -** <user COLUMns>, +** <user columns>, ** <table name> HIDDEN, ** docid HIDDEN, ** <langid> HIDDEN @@ -3516,7 +5351,7 @@ int sqlite3Fts3UpdateMethod( nChng++; } - if( p->bHasStat ){ + if( p->bFts4 ){ fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); } diff --git a/lib/libsqlite3/ext/fts3/tool/fts3view.c b/lib/libsqlite3/ext/fts3/tool/fts3view.c new file mode 100644 index 00000000000..479ae9868d4 --- /dev/null +++ b/lib/libsqlite3/ext/fts3/tool/fts3view.c @@ -0,0 +1,874 @@ +/* +** This program is a debugging and analysis utility that displays +** information about an FTS3 or FTS4 index. +** +** Link this program against the SQLite3 amalgamation with the +** SQLITE_ENABLE_FTS4 compile-time option. Then run it as: +** +** fts3view DATABASE +** +** to get a list of all FTS3/4 tables in DATABASE, or do +** +** fts3view DATABASE TABLE COMMAND .... +** +** to see various aspects of the TABLE table. Type fts3view with no +** arguments for a list of available COMMANDs. +*/ +#include <stdio.h> +#include <stdarg.h> +#include <stdlib.h> +#include <string.h> +#include <ctype.h> +#include "sqlite3.h" + +/* +** Extra command-line arguments: +*/ +int nExtra; +char **azExtra; + +/* +** Look for a command-line argument. +*/ +const char *findOption(const char *zName, int hasArg, const char *zDefault){ + int i; + const char *zResult = zDefault; + for(i=0; i<nExtra; i++){ + const char *z = azExtra[i]; + while( z[0]=='-' ) z++; + if( strcmp(z, zName)==0 ){ + int j = 1; + if( hasArg==0 || i==nExtra-1 ) j = 0; + zResult = azExtra[i+j]; + while( i+j<nExtra ){ + azExtra[i] = azExtra[i+j+1]; + i++; + } + break; + } + } + return zResult; +} + + +/* +** Prepare an SQL query +*/ +static sqlite3_stmt *prepare(sqlite3 *db, const char *zFormat, ...){ + va_list ap; + char *zSql; + sqlite3_stmt *pStmt; + int rc; + + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc ){ + fprintf(stderr, "Error: %s\nSQL: %s\n", sqlite3_errmsg(db), zSql); + exit(1); + } + sqlite3_free(zSql); + return pStmt; +} + +/* +** Run an SQL statement +*/ +static int runSql(sqlite3 *db, const char *zFormat, ...){ + va_list ap; + char *zSql; + int rc; + + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + rc = sqlite3_exec(db, zSql, 0, 0, 0); + va_end(ap); + return rc; +} + +/* +** Show the table schema +*/ +static void showSchema(sqlite3 *db, const char *zTab){ + sqlite3_stmt *pStmt; + pStmt = prepare(db, + "SELECT sql FROM sqlite_master" + " WHERE name LIKE '%q%%'" + " ORDER BY 1", + zTab); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + printf("%s;\n", sqlite3_column_text(pStmt, 0)); + } + sqlite3_finalize(pStmt); + pStmt = prepare(db, "PRAGMA page_size"); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + printf("PRAGMA page_size=%s;\n", sqlite3_column_text(pStmt, 0)); + } + sqlite3_finalize(pStmt); + pStmt = prepare(db, "PRAGMA journal_mode"); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + printf("PRAGMA journal_mode=%s;\n", sqlite3_column_text(pStmt, 0)); + } + sqlite3_finalize(pStmt); + pStmt = prepare(db, "PRAGMA auto_vacuum"); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + const char *zType = "???"; + switch( sqlite3_column_int(pStmt, 0) ){ + case 0: zType = "OFF"; break; + case 1: zType = "FULL"; break; + case 2: zType = "INCREMENTAL"; break; + } + printf("PRAGMA auto_vacuum=%s;\n", zType); + } + sqlite3_finalize(pStmt); + pStmt = prepare(db, "PRAGMA encoding"); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + printf("PRAGMA encoding=%s;\n", sqlite3_column_text(pStmt, 0)); + } + sqlite3_finalize(pStmt); +} + +/* +** Read a 64-bit variable-length integer from memory starting at p[0]. +** Return the number of bytes read, or 0 on error. +** The value is stored in *v. +*/ +int getVarint(const unsigned char *p, sqlite_int64 *v){ + const unsigned char *q = p; + sqlite_uint64 x = 0, y = 1; + while( (*q&0x80)==0x80 && q-(unsigned char *)p<9 ){ + x += y * (*q++ & 0x7f); + y <<= 7; + } + x += y * (*q++); + *v = (sqlite_int64) x; + return (int) (q - (unsigned char *)p); +} + + +/* Show the content of the %_stat table +*/ +static void showStat(sqlite3 *db, const char *zTab){ + sqlite3_stmt *pStmt; + pStmt = prepare(db, "SELECT id, value FROM '%q_stat'", zTab); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + printf("stat[%d] =", sqlite3_column_int(pStmt, 0)); + switch( sqlite3_column_type(pStmt, 1) ){ + case SQLITE_INTEGER: { + printf(" %d\n", sqlite3_column_int(pStmt, 1)); + break; + } + case SQLITE_BLOB: { + unsigned char *x = (unsigned char*)sqlite3_column_blob(pStmt, 1); + int len = sqlite3_column_bytes(pStmt, 1); + int i = 0; + sqlite3_int64 v; + while( i<len ){ + i += getVarint(x, &v); + printf(" %lld", v); + } + printf("\n"); + break; + } + } + } + sqlite3_finalize(pStmt); +} + +/* +** Report on the vocabulary. This creates an fts4aux table with a random +** name, but deletes it in the end. +*/ +static void showVocabulary(sqlite3 *db, const char *zTab){ + char *zAux; + sqlite3_uint64 r; + sqlite3_stmt *pStmt; + int nDoc = 0; + int nToken = 0; + int nOccurrence = 0; + int nTop; + int n, i; + + sqlite3_randomness(sizeof(r), &r); + zAux = sqlite3_mprintf("viewer_%llx", zTab, r); + runSql(db, "BEGIN"); + pStmt = prepare(db, "SELECT count(*) FROM %Q", zTab); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + nDoc = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + printf("Number of documents...................... %9d\n", nDoc); + + runSql(db, "CREATE VIRTUAL TABLE %s USING fts4aux(%Q)", zAux, zTab); + pStmt = prepare(db, + "SELECT count(*), sum(occurrences) FROM %s WHERE col='*'", + zAux); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + nToken = sqlite3_column_int(pStmt, 0); + nOccurrence = sqlite3_column_int(pStmt, 1); + } + sqlite3_finalize(pStmt); + printf("Total tokens in all documents............ %9d\n", nOccurrence); + printf("Total number of distinct tokens.......... %9d\n", nToken); + if( nToken==0 ) goto end_vocab; + + n = 0; + pStmt = prepare(db, "SELECT count(*) FROM %s" + " WHERE col='*' AND occurrences==1", zAux); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + n = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + printf("Tokens used exactly once................. %9d %5.2f%%\n", + n, n*100.0/nToken); + + n = 0; + pStmt = prepare(db, "SELECT count(*) FROM %s" + " WHERE col='*' AND documents==1", zAux); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + n = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + printf("Tokens used in only one document......... %9d %5.2f%%\n", + n, n*100.0/nToken); + + if( nDoc>=2000 ){ + n = 0; + pStmt = prepare(db, "SELECT count(*) FROM %s" + " WHERE col='*' AND occurrences<=%d", zAux, nDoc/1000); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + n = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + printf("Tokens used in 0.1%% or less of docs...... %9d %5.2f%%\n", + n, n*100.0/nToken); + } + + if( nDoc>=200 ){ + n = 0; + pStmt = prepare(db, "SELECT count(*) FROM %s" + " WHERE col='*' AND occurrences<=%d", zAux, nDoc/100); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + n = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + printf("Tokens used in 1%% or less of docs........ %9d %5.2f%%\n", + n, n*100.0/nToken); + } + + nTop = atoi(findOption("top", 1, "25")); + printf("The %d most common tokens:\n", nTop); + pStmt = prepare(db, + "SELECT term, documents FROM %s" + " WHERE col='*'" + " ORDER BY documents DESC, term" + " LIMIT %d", zAux, nTop); + i = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + i++; + n = sqlite3_column_int(pStmt, 1); + printf(" %2d. %-30s %9d docs %5.2f%%\n", i, + sqlite3_column_text(pStmt, 0), n, n*100.0/nDoc); + } + sqlite3_finalize(pStmt); + +end_vocab: + runSql(db, "ROLLBACK"); + sqlite3_free(zAux); +} + +/* +** Report on the number and sizes of segments +*/ +static void showSegmentStats(sqlite3 *db, const char *zTab){ + sqlite3_stmt *pStmt; + int nSeg = 0; + sqlite3_int64 szSeg = 0, mxSeg = 0; + int nIdx = 0; + sqlite3_int64 szIdx = 0, mxIdx = 0; + int nRoot = 0; + sqlite3_int64 szRoot = 0, mxRoot = 0; + sqlite3_int64 mx; + int nLeaf; + int n; + int pgsz; + int mxLevel; + int i; + + pStmt = prepare(db, + "SELECT count(*), sum(length(block)), max(length(block))" + " FROM '%q_segments'", + zTab); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + nSeg = sqlite3_column_int(pStmt, 0); + szSeg = sqlite3_column_int64(pStmt, 1); + mxSeg = sqlite3_column_int64(pStmt, 2); + } + sqlite3_finalize(pStmt); + pStmt = prepare(db, + "SELECT count(*), sum(length(block)), max(length(block))" + " FROM '%q_segments' a JOIN '%q_segdir' b" + " WHERE a.blockid BETWEEN b.leaves_end_block+1 AND b.end_block", + zTab, zTab); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + nIdx = sqlite3_column_int(pStmt, 0); + szIdx = sqlite3_column_int64(pStmt, 1); + mxIdx = sqlite3_column_int64(pStmt, 2); + } + sqlite3_finalize(pStmt); + pStmt = prepare(db, + "SELECT count(*), sum(length(root)), max(length(root))" + " FROM '%q_segdir'", + zTab); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + nRoot = sqlite3_column_int(pStmt, 0); + szRoot = sqlite3_column_int64(pStmt, 1); + mxRoot = sqlite3_column_int64(pStmt, 2); + } + sqlite3_finalize(pStmt); + + printf("Number of segments....................... %9d\n", nSeg+nRoot); + printf("Number of leaf segments.................. %9d\n", nSeg-nIdx); + printf("Number of index segments................. %9d\n", nIdx); + printf("Number of root segments.................. %9d\n", nRoot); + printf("Total size of all segments............... %9lld\n", szSeg+szRoot); + printf("Total size of all leaf segments.......... %9lld\n", szSeg-szIdx); + printf("Total size of all index segments......... %9lld\n", szIdx); + printf("Total size of all root segments.......... %9lld\n", szRoot); + if( nSeg>0 ){ + printf("Average size of all segments............. %11.1f\n", + (double)(szSeg+szRoot)/(double)(nSeg+nRoot)); + printf("Average size of leaf segments............ %11.1f\n", + (double)(szSeg-szIdx)/(double)(nSeg-nIdx)); + } + if( nIdx>0 ){ + printf("Average size of index segments........... %11.1f\n", + (double)szIdx/(double)nIdx); + } + if( nRoot>0 ){ + printf("Average size of root segments............ %11.1f\n", + (double)szRoot/(double)nRoot); + } + mx = mxSeg; + if( mx<mxRoot ) mx = mxRoot; + printf("Maximum segment size..................... %9lld\n", mx); + printf("Maximum index segment size............... %9lld\n", mxIdx); + printf("Maximum root segment size................ %9lld\n", mxRoot); + + pStmt = prepare(db, "PRAGMA page_size"); + pgsz = 1024; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + pgsz = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + printf("Database page size....................... %9d\n", pgsz); + pStmt = prepare(db, + "SELECT count(*)" + " FROM '%q_segments' a JOIN '%q_segdir' b" + " WHERE a.blockid BETWEEN b.start_block AND b.leaves_end_block" + " AND length(a.block)>%d", + zTab, zTab, pgsz-45); + n = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + n = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + nLeaf = nSeg - nIdx; + printf("Leaf segments larger than %5d bytes.... %9d %5.2f%%\n", + pgsz-45, n, n*100.0/nLeaf); + + pStmt = prepare(db, "SELECT max(level%%1024) FROM '%q_segdir'", zTab); + mxLevel = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + mxLevel = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + + for(i=0; i<=mxLevel; i++){ + pStmt = prepare(db, + "SELECT count(*), sum(len), avg(len), max(len), sum(len>%d)," + " count(distinct idx)" + " FROM (SELECT length(a.block) AS len, idx" + " FROM '%q_segments' a JOIN '%q_segdir' b" + " WHERE (a.blockid BETWEEN b.start_block" + " AND b.leaves_end_block)" + " AND (b.level%%1024)==%d)", + pgsz-45, zTab, zTab, i); + if( sqlite3_step(pStmt)==SQLITE_ROW + && (nLeaf = sqlite3_column_int(pStmt, 0))>0 + ){ + int nIdx = sqlite3_column_int(pStmt, 5); + sqlite3_int64 sz; + printf("For level %d:\n", i); + printf(" Number of indexes...................... %9d\n", nIdx); + printf(" Number of leaf segments................ %9d\n", nLeaf); + if( nIdx>1 ){ + printf(" Average leaf segments per index........ %11.1f\n", + (double)nLeaf/(double)nIdx); + } + printf(" Total size of all leaf segments........ %9lld\n", + (sz = sqlite3_column_int64(pStmt, 1))); + printf(" Average size of leaf segments.......... %11.1f\n", + sqlite3_column_double(pStmt, 2)); + if( nIdx>1 ){ + printf(" Average leaf segment size per index.... %11.1f\n", + (double)sz/(double)nIdx); + } + printf(" Maximum leaf segment size.............. %9lld\n", + sqlite3_column_int64(pStmt, 3)); + n = sqlite3_column_int(pStmt, 4); + printf(" Leaf segments larger than %5d bytes.. %9d %5.2f%%\n", + pgsz-45, n, n*100.0/nLeaf); + } + sqlite3_finalize(pStmt); + } +} + +/* +** Print a single "tree" line of the segdir map output. +*/ +static void printTreeLine(sqlite3_int64 iLower, sqlite3_int64 iUpper){ + printf(" tree %9lld", iLower); + if( iUpper>iLower ){ + printf(" thru %9lld (%lld blocks)", iUpper, iUpper-iLower+1); + } + printf("\n"); +} + +/* +** Check to see if the block of a %_segments entry is NULL. +*/ +static int isNullSegment(sqlite3 *db, const char *zTab, sqlite3_int64 iBlockId){ + sqlite3_stmt *pStmt; + int rc = 1; + + pStmt = prepare(db, "SELECT block IS NULL FROM '%q_segments'" + " WHERE blockid=%lld", zTab, iBlockId); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + rc = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + return rc; +} + +/* +** Show a map of segments derived from the %_segdir table. +*/ +static void showSegdirMap(sqlite3 *db, const char *zTab){ + int mxIndex, iIndex; + sqlite3_stmt *pStmt = 0; + sqlite3_stmt *pStmt2 = 0; + int prevLevel; + + pStmt = prepare(db, "SELECT max(level/1024) FROM '%q_segdir'", zTab); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + mxIndex = sqlite3_column_int(pStmt, 0); + }else{ + mxIndex = 0; + } + sqlite3_finalize(pStmt); + + printf("Number of inverted indices............... %3d\n", mxIndex+1); + pStmt = prepare(db, + "SELECT level, idx, start_block, leaves_end_block, end_block, rowid" + " FROM '%q_segdir'" + " WHERE level/1024==?" + " ORDER BY level DESC, idx", + zTab); + pStmt2 = prepare(db, + "SELECT blockid FROM '%q_segments'" + " WHERE blockid BETWEEN ? AND ? ORDER BY blockid", + zTab); + for(iIndex=0; iIndex<=mxIndex; iIndex++){ + if( mxIndex>0 ){ + printf("**************************** Index %d " + "****************************\n", iIndex); + } + sqlite3_bind_int(pStmt, 1, iIndex); + prevLevel = -1; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + int iLevel = sqlite3_column_int(pStmt, 0)%1024; + int iIdx = sqlite3_column_int(pStmt, 1); + sqlite3_int64 iStart = sqlite3_column_int64(pStmt, 2); + sqlite3_int64 iLEnd = sqlite3_column_int64(pStmt, 3); + sqlite3_int64 iEnd = sqlite3_column_int64(pStmt, 4); + char rtag[20]; + if( iLevel!=prevLevel ){ + printf("level %2d idx %2d", iLevel, iIdx); + prevLevel = iLevel; + }else{ + printf(" idx %2d", iIdx); + } + sqlite3_snprintf(sizeof(rtag), rtag, "r%lld", + sqlite3_column_int64(pStmt,5)); + printf(" root %9s\n", rtag); + if( iLEnd>iStart ){ + sqlite3_int64 iLower, iPrev, iX; + if( iLEnd+1<=iEnd ){ + sqlite3_bind_int64(pStmt2, 1, iLEnd+1); + sqlite3_bind_int64(pStmt2, 2, iEnd); + iLower = -1; + while( sqlite3_step(pStmt2)==SQLITE_ROW ){ + iX = sqlite3_column_int64(pStmt2, 0); + if( iLower<0 ){ + iLower = iPrev = iX; + }else if( iX==iPrev+1 ){ + iPrev = iX; + }else{ + printTreeLine(iLower, iPrev); + iLower = iPrev = iX; + } + } + sqlite3_reset(pStmt2); + if( iLower>=0 ){ + if( iLower==iPrev && iLower==iEnd + && isNullSegment(db,zTab,iLower) + ){ + printf(" null %9lld\n", iLower); + }else{ + printTreeLine(iLower, iPrev); + } + } + } + printf(" leaves %9lld thru %9lld (%lld blocks)\n", + iStart, iLEnd, iLEnd - iStart + 1); + } + } + sqlite3_reset(pStmt); + } + sqlite3_finalize(pStmt); + sqlite3_finalize(pStmt2); +} + +/* +** Decode a single segment block and display the results on stdout. +*/ +static void decodeSegment( + const unsigned char *aData, /* Content to print */ + int nData /* Number of bytes of content */ +){ + sqlite3_int64 iChild; + sqlite3_int64 iPrefix; + sqlite3_int64 nTerm; + sqlite3_int64 n; + sqlite3_int64 iDocsz; + int iHeight; + int i = 0; + int cnt = 0; + char zTerm[1000]; + + i += getVarint(aData, &n); + iHeight = (int)n; + printf("height: %d\n", iHeight); + if( iHeight>0 ){ + i += getVarint(aData+i, &iChild); + printf("left-child: %lld\n", iChild); + } + while( i<nData ){ + if( (cnt++)>0 ){ + i += getVarint(aData+i, &iPrefix); + }else{ + iPrefix = 0; + } + i += getVarint(aData+i, &nTerm); + if( iPrefix+nTerm+1 >= sizeof(zTerm) ){ + fprintf(stderr, "term to long\n"); + exit(1); + } + memcpy(zTerm+iPrefix, aData+i, nTerm); + zTerm[iPrefix+nTerm] = 0; + i += nTerm; + if( iHeight==0 ){ + i += getVarint(aData+i, &iDocsz); + printf("term: %-25s doclist %7lld bytes offset %d\n", zTerm, iDocsz, i); + i += iDocsz; + }else{ + printf("term: %-25s child %lld\n", zTerm, ++iChild); + } + } +} + + +/* +** Print a a blob as hex and ascii. +*/ +static void printBlob( + const unsigned char *aData, /* Content to print */ + int nData /* Number of bytes of content */ +){ + int i, j; + const char *zOfstFmt; + const int perLine = 16; + + if( (nData&~0xfff)==0 ){ + zOfstFmt = " %03x: "; + }else if( (nData&~0xffff)==0 ){ + zOfstFmt = " %04x: "; + }else if( (nData&~0xfffff)==0 ){ + zOfstFmt = " %05x: "; + }else if( (nData&~0xffffff)==0 ){ + zOfstFmt = " %06x: "; + }else{ + zOfstFmt = " %08x: "; + } + + for(i=0; i<nData; i += perLine){ + fprintf(stdout, zOfstFmt, i); + for(j=0; j<perLine; j++){ + if( i+j>nData ){ + fprintf(stdout, " "); + }else{ + fprintf(stdout,"%02x ", aData[i+j]); + } + } + for(j=0; j<perLine; j++){ + if( i+j>nData ){ + fprintf(stdout, " "); + }else{ + fprintf(stdout,"%c", isprint(aData[i+j]) ? aData[i+j] : '.'); + } + } + fprintf(stdout,"\n"); + } +} + +/* +** Convert text to a 64-bit integer +*/ +static sqlite3_int64 atoi64(const char *z){ + sqlite3_int64 v = 0; + while( z[0]>='0' && z[0]<='9' ){ + v = v*10 + z[0] - '0'; + z++; + } + return v; +} + +/* +** Return a prepared statement which, when stepped, will return in its +** first column the blob associated with segment zId. If zId begins with +** 'r' then it is a rowid of a %_segdir entry. Otherwise it is a +** %_segment entry. +*/ +static sqlite3_stmt *prepareToGetSegment( + sqlite3 *db, /* The database */ + const char *zTab, /* The FTS3/4 table name */ + const char *zId /* ID of the segment to open */ +){ + sqlite3_stmt *pStmt; + if( zId[0]=='r' ){ + pStmt = prepare(db, "SELECT root FROM '%q_segdir' WHERE rowid=%lld", + zTab, atoi64(zId+1)); + }else{ + pStmt = prepare(db, "SELECT block FROM '%q_segments' WHERE blockid=%lld", + zTab, atoi64(zId)); + } + return pStmt; +} + +/* +** Print the content of a segment or of the root of a segdir. The segment +** or root is identified by azExtra[0]. If the first character of azExtra[0] +** is 'r' then the remainder is the integer rowid of the %_segdir entry. +** If the first character of azExtra[0] is not 'r' then, then all of +** azExtra[0] is an integer which is the block number. +** +** If the --raw option is present in azExtra, then a hex dump is provided. +** Otherwise a decoding is shown. +*/ +static void showSegment(sqlite3 *db, const char *zTab){ + const unsigned char *aData; + int nData; + sqlite3_stmt *pStmt; + + pStmt = prepareToGetSegment(db, zTab, azExtra[0]); + if( sqlite3_step(pStmt)!=SQLITE_ROW ){ + sqlite3_finalize(pStmt); + return; + } + nData = sqlite3_column_bytes(pStmt, 0); + aData = sqlite3_column_blob(pStmt, 0); + printf("Segment %s of size %d bytes:\n", azExtra[0], nData); + if( findOption("raw", 0, 0)!=0 ){ + printBlob(aData, nData); + }else{ + decodeSegment(aData, nData); + } + sqlite3_finalize(pStmt); +} + +/* +** Decode a single doclist and display the results on stdout. +*/ +static void decodeDoclist( + const unsigned char *aData, /* Content to print */ + int nData /* Number of bytes of content */ +){ + sqlite3_int64 iPrevDocid = 0; + sqlite3_int64 iDocid; + sqlite3_int64 iPos; + sqlite3_int64 iPrevPos = 0; + sqlite3_int64 iCol; + int i = 0; + + while( i<nData ){ + i += getVarint(aData+i, &iDocid); + printf("docid %lld col0", iDocid+iPrevDocid); + iPrevDocid += iDocid; + iPrevPos = 0; + while( 1 ){ + i += getVarint(aData+i, &iPos); + if( iPos==1 ){ + i += getVarint(aData+i, &iCol); + printf(" col%lld", iCol); + iPrevPos = 0; + }else if( iPos==0 ){ + printf("\n"); + break; + }else{ + iPrevPos += iPos - 2; + printf(" %lld", iPrevPos); + } + } + } +} + + +/* +** Print the content of a doclist. The segment or segdir-root is +** identified by azExtra[0]. If the first character of azExtra[0] +** is 'r' then the remainder is the integer rowid of the %_segdir entry. +** If the first character of azExtra[0] is not 'r' then, then all of +** azExtra[0] is an integer which is the block number. The offset +** into the segment is identified by azExtra[1]. The size of the doclist +** is azExtra[2]. +** +** If the --raw option is present in azExtra, then a hex dump is provided. +** Otherwise a decoding is shown. +*/ +static void showDoclist(sqlite3 *db, const char *zTab){ + const unsigned char *aData; + sqlite3_int64 offset, nData; + sqlite3_stmt *pStmt; + + offset = atoi64(azExtra[1]); + nData = atoi64(azExtra[2]); + pStmt = prepareToGetSegment(db, zTab, azExtra[0]); + if( sqlite3_step(pStmt)!=SQLITE_ROW ){ + sqlite3_finalize(pStmt); + return; + } + aData = sqlite3_column_blob(pStmt, 0); + printf("Doclist at %s offset %lld of size %lld bytes:\n", + azExtra[0], offset, nData); + if( findOption("raw", 0, 0)!=0 ){ + printBlob(aData+offset, nData); + }else{ + decodeDoclist(aData+offset, nData); + } + sqlite3_finalize(pStmt); +} + +/* +** Show the top N largest segments +*/ +static void listBigSegments(sqlite3 *db, const char *zTab){ + int nTop, i; + sqlite3_stmt *pStmt; + sqlite3_int64 sz; + sqlite3_int64 id; + + nTop = atoi(findOption("top", 1, "25")); + printf("The %d largest segments:\n", nTop); + pStmt = prepare(db, + "SELECT blockid, length(block) AS len FROM '%q_segments'" + " ORDER BY 2 DESC, 1" + " LIMIT %d", zTab, nTop); + i = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + i++; + id = sqlite3_column_int64(pStmt, 0); + sz = sqlite3_column_int64(pStmt, 1); + printf(" %2d. %9lld size %lld\n", i, id, sz); + } + sqlite3_finalize(pStmt); +} + + + +static void usage(const char *argv0){ + fprintf(stderr, "Usage: %s DATABASE\n" + " or: %s DATABASE FTS3TABLE ARGS...\n", argv0, argv0); + fprintf(stderr, + "ARGS:\n" + " big-segments [--top N] show the largest segments\n" + " doclist BLOCKID OFFSET SIZE [--raw] Decode a doclist\n" + " schema FTS table schema\n" + " segdir directory of segments\n" + " segment BLOCKID [--raw] content of a segment\n" + " segment-stats info on segment sizes\n" + " stat the %%_stat table\n" + " vocabulary [--top N] document vocabulary\n" + ); + exit(1); +} + +int main(int argc, char **argv){ + sqlite3 *db; + int rc; + const char *zTab; + const char *zCmd; + + if( argc<2 ) usage(argv[0]); + rc = sqlite3_open(argv[1], &db); + if( rc ){ + fprintf(stderr, "Cannot open %s\n", argv[1]); + exit(1); + } + if( argc==2 ){ + sqlite3_stmt *pStmt; + int cnt = 0; + pStmt = prepare(db, "SELECT b.sql" + " FROM sqlite_master a, sqlite_master b" + " WHERE a.name GLOB '*_segdir'" + " AND b.name=substr(a.name,1,length(a.name)-7)" + " ORDER BY 1"); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + cnt++; + printf("%s;\n", sqlite3_column_text(pStmt, 0)); + } + sqlite3_finalize(pStmt); + if( cnt==0 ){ + printf("/* No FTS3/4 tables found in database %s */\n", argv[1]); + } + return 0; + } + if( argc<4 ) usage(argv[0]); + zTab = argv[2]; + zCmd = argv[3]; + nExtra = argc-4; + azExtra = argv+4; + if( strcmp(zCmd,"big-segments")==0 ){ + listBigSegments(db, zTab); + }else if( strcmp(zCmd,"doclist")==0 ){ + if( argc<7 ) usage(argv[0]); + showDoclist(db, zTab); + }else if( strcmp(zCmd,"schema")==0 ){ + showSchema(db, zTab); + }else if( strcmp(zCmd,"segdir")==0 ){ + showSegdirMap(db, zTab); + }else if( strcmp(zCmd,"segment")==0 ){ + if( argc<5 ) usage(argv[0]); + showSegment(db, zTab); + }else if( strcmp(zCmd,"segment-stats")==0 ){ + showSegmentStats(db, zTab); + }else if( strcmp(zCmd,"stat")==0 ){ + showStat(db, zTab); + }else if( strcmp(zCmd,"vocabulary")==0 ){ + showVocabulary(db, zTab); + }else{ + usage(argv[0]); + } + return 0; +} diff --git a/lib/libsqlite3/ext/rtree/rtree.c b/lib/libsqlite3/ext/rtree/rtree.c index ce76e61f08d..d6cdde9fc45 100644 --- a/lib/libsqlite3/ext/rtree/rtree.c +++ b/lib/libsqlite3/ext/rtree/rtree.c @@ -183,6 +183,19 @@ struct Rtree { #define RTREE_COORD_INT32 1 /* +** If SQLITE_RTREE_INT_ONLY is defined, then this virtual table will +** only deal with integer coordinates. No floating point operations +** will be done. +*/ +#ifdef SQLITE_RTREE_INT_ONLY + typedef sqlite3_int64 RtreeDValue; /* High accuracy coordinate */ + typedef int RtreeValue; /* Low accuracy coordinate */ +#else + typedef double RtreeDValue; /* High accuracy coordinate */ + typedef float RtreeValue; /* Low accuracy coordinate */ +#endif + +/* ** The minimum number of cells allowed for a node is a third of the ** maximum. In Gutman's notation: ** @@ -217,20 +230,25 @@ struct RtreeCursor { }; union RtreeCoord { - float f; + RtreeValue f; int i; }; /* ** The argument is an RtreeCoord. Return the value stored within the RtreeCoord -** formatted as a double. This macro assumes that local variable pRtree points -** to the Rtree structure associated with the RtreeCoord. +** formatted as a RtreeDValue (double or int64). This macro assumes that local +** variable pRtree points to the Rtree structure associated with the +** RtreeCoord. */ -#define DCOORD(coord) ( \ - (pRtree->eCoordType==RTREE_COORD_REAL32) ? \ - ((double)coord.f) : \ - ((double)coord.i) \ -) +#ifdef SQLITE_RTREE_INT_ONLY +# define DCOORD(coord) ((RtreeDValue)coord.i) +#else +# define DCOORD(coord) ( \ + (pRtree->eCoordType==RTREE_COORD_REAL32) ? \ + ((double)coord.f) : \ + ((double)coord.i) \ + ) +#endif /* ** A search constraint. @@ -238,8 +256,8 @@ union RtreeCoord { struct RtreeConstraint { int iCoord; /* Index of constrained coordinate */ int op; /* Constraining operation */ - double rValue; /* Constraint value. */ - int (*xGeom)(sqlite3_rtree_geometry *, int, double *, int *); + RtreeDValue rValue; /* Constraint value. */ + int (*xGeom)(sqlite3_rtree_geometry*, int, RtreeDValue*, int*); sqlite3_rtree_geometry *pGeom; /* Constraint callback argument for a MATCH */ }; @@ -287,10 +305,10 @@ struct RtreeCell { */ struct RtreeMatchArg { u32 magic; /* Always RTREE_GEOMETRY_MAGIC */ - int (*xGeom)(sqlite3_rtree_geometry *, int, double *, int *); + int (*xGeom)(sqlite3_rtree_geometry *, int, RtreeDValue*, int *); void *pContext; int nParam; - double aParam[1]; + RtreeDValue aParam[1]; }; /* @@ -302,7 +320,7 @@ struct RtreeMatchArg { ** the geometry callback function). */ struct RtreeGeomCallback { - int (*xGeom)(sqlite3_rtree_geometry *, int, double *, int *); + int (*xGeom)(sqlite3_rtree_geometry*, int, RtreeDValue*, int*); void *pContext; }; @@ -868,7 +886,7 @@ static int testRtreeGeom( int *pbRes /* OUT: Test result */ ){ int i; - double aCoord[RTREE_MAX_DIMENSIONS*2]; + RtreeDValue aCoord[RTREE_MAX_DIMENSIONS*2]; int nCoord = pRtree->nDim*2; assert( pConstraint->op==RTREE_MATCH ); @@ -898,8 +916,8 @@ static int testRtreeCell(Rtree *pRtree, RtreeCursor *pCursor, int *pbEof){ nodeGetCell(pRtree, pCursor->pNode, pCursor->iCell, &cell); for(ii=0; bRes==0 && ii<pCursor->nConstraint; ii++){ RtreeConstraint *p = &pCursor->aConstraint[ii]; - double cell_min = DCOORD(cell.aCoord[(p->iCoord>>1)*2]); - double cell_max = DCOORD(cell.aCoord[(p->iCoord>>1)*2+1]); + RtreeDValue cell_min = DCOORD(cell.aCoord[(p->iCoord>>1)*2]); + RtreeDValue cell_max = DCOORD(cell.aCoord[(p->iCoord>>1)*2+1]); assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_MATCH @@ -951,7 +969,7 @@ static int testRtreeEntry(Rtree *pRtree, RtreeCursor *pCursor, int *pbEof){ nodeGetCell(pRtree, pCursor->pNode, pCursor->iCell, &cell); for(ii=0; ii<pCursor->nConstraint; ii++){ RtreeConstraint *p = &pCursor->aConstraint[ii]; - double coord = DCOORD(cell.aCoord[p->iCoord]); + RtreeDValue coord = DCOORD(cell.aCoord[p->iCoord]); int res; assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_MATCH @@ -1149,9 +1167,12 @@ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ }else{ RtreeCoord c; nodeGetCoord(pRtree, pCsr->pNode, pCsr->iCell, i-1, &c); +#ifndef SQLITE_RTREE_INT_ONLY if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ sqlite3_result_double(ctx, c.f); - }else{ + }else +#endif + { assert( pRtree->eCoordType==RTREE_COORD_INT32 ); sqlite3_result_int(ctx, c.i); } @@ -1198,7 +1219,7 @@ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ /* Check that the blob is roughly the right size. */ nBlob = sqlite3_value_bytes(pValue); if( nBlob<(int)sizeof(RtreeMatchArg) - || ((nBlob-sizeof(RtreeMatchArg))%sizeof(double))!=0 + || ((nBlob-sizeof(RtreeMatchArg))%sizeof(RtreeDValue))!=0 ){ return SQLITE_ERROR; } @@ -1212,7 +1233,7 @@ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ memcpy(p, sqlite3_value_blob(pValue), nBlob); if( p->magic!=RTREE_GEOMETRY_MAGIC - || nBlob!=(int)(sizeof(RtreeMatchArg) + (p->nParam-1)*sizeof(double)) + || nBlob!=(int)(sizeof(RtreeMatchArg) + (p->nParam-1)*sizeof(RtreeDValue)) ){ sqlite3_free(pGeom); return SQLITE_ERROR; @@ -1284,7 +1305,11 @@ static int rtreeFilter( break; } }else{ +#ifdef SQLITE_RTREE_INT_ONLY + p->rValue = sqlite3_value_int64(argv[ii]); +#else p->rValue = sqlite3_value_double(argv[ii]); +#endif } } } @@ -1418,11 +1443,11 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ /* ** Return the N-dimensional volumn of the cell stored in *p. */ -static float cellArea(Rtree *pRtree, RtreeCell *p){ - float area = 1.0; +static RtreeDValue cellArea(Rtree *pRtree, RtreeCell *p){ + RtreeDValue area = (RtreeDValue)1; int ii; for(ii=0; ii<(pRtree->nDim*2); ii+=2){ - area = (float)(area * (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii]))); + area = (area * (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii]))); } return area; } @@ -1431,11 +1456,11 @@ static float cellArea(Rtree *pRtree, RtreeCell *p){ ** Return the margin length of cell p. The margin length is the sum ** of the objects size in each dimension. */ -static float cellMargin(Rtree *pRtree, RtreeCell *p){ - float margin = 0.0; +static RtreeDValue cellMargin(Rtree *pRtree, RtreeCell *p){ + RtreeDValue margin = (RtreeDValue)0; int ii; for(ii=0; ii<(pRtree->nDim*2); ii+=2){ - margin += (float)(DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii])); + margin += (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii])); } return margin; } @@ -1480,8 +1505,8 @@ static int cellContains(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ /* ** Return the amount cell p would grow by if it were unioned with pCell. */ -static float cellGrowth(Rtree *pRtree, RtreeCell *p, RtreeCell *pCell){ - float area; +static RtreeDValue cellGrowth(Rtree *pRtree, RtreeCell *p, RtreeCell *pCell){ + RtreeDValue area; RtreeCell cell; memcpy(&cell, p, sizeof(RtreeCell)); area = cellArea(pRtree, &cell); @@ -1490,7 +1515,7 @@ static float cellGrowth(Rtree *pRtree, RtreeCell *p, RtreeCell *pCell){ } #if VARIANT_RSTARTREE_CHOOSESUBTREE || VARIANT_RSTARTREE_SPLIT -static float cellOverlap( +static RtreeDValue cellOverlap( Rtree *pRtree, RtreeCell *p, RtreeCell *aCell, @@ -1498,7 +1523,7 @@ static float cellOverlap( int iExclude ){ int ii; - float overlap = 0.0; + RtreeDValue overlap = 0.0; for(ii=0; ii<nCell; ii++){ #if VARIANT_RSTARTREE_CHOOSESUBTREE if( ii!=iExclude ) @@ -1508,10 +1533,9 @@ static float cellOverlap( #endif { int jj; - float o = 1.0; + RtreeDValue o = (RtreeDValue)1; for(jj=0; jj<(pRtree->nDim*2); jj+=2){ - double x1; - double x2; + RtreeDValue x1, x2; x1 = MAX(DCOORD(p->aCoord[jj]), DCOORD(aCell[ii].aCoord[jj])); x2 = MIN(DCOORD(p->aCoord[jj+1]), DCOORD(aCell[ii].aCoord[jj+1])); @@ -1520,7 +1544,7 @@ static float cellOverlap( o = 0.0; break; }else{ - o = o * (float)(x2-x1); + o = o * (x2-x1); } } overlap += o; @@ -1531,7 +1555,7 @@ static float cellOverlap( #endif #if VARIANT_RSTARTREE_CHOOSESUBTREE -static float cellOverlapEnlargement( +static RtreeDValue cellOverlapEnlargement( Rtree *pRtree, RtreeCell *p, RtreeCell *pInsert, @@ -1539,12 +1563,11 @@ static float cellOverlapEnlargement( int nCell, int iExclude ){ - double before; - double after; + RtreeDValue before, after; before = cellOverlap(pRtree, p, aCell, nCell, iExclude); cellUnion(pRtree, p, pInsert); after = cellOverlap(pRtree, p, aCell, nCell, iExclude); - return (float)(after-before); + return (after-before); } #endif @@ -1568,11 +1591,11 @@ static int ChooseLeaf( int iCell; sqlite3_int64 iBest = 0; - float fMinGrowth = 0.0; - float fMinArea = 0.0; + RtreeDValue fMinGrowth = 0.0; + RtreeDValue fMinArea = 0.0; #if VARIANT_RSTARTREE_CHOOSESUBTREE - float fMinOverlap = 0.0; - float overlap; + RtreeDValue fMinOverlap = 0.0; + RtreeDValue overlap; #endif int nCell = NCELL(pNode); @@ -1603,8 +1626,8 @@ static int ChooseLeaf( */ for(iCell=0; iCell<nCell; iCell++){ int bBest = 0; - float growth; - float area; + RtreeDValue growth; + RtreeDValue area; nodeGetCell(pRtree, pNode, iCell, &cell); growth = cellGrowth(pRtree, &cell, pCell); area = cellArea(pRtree, &cell); @@ -1731,7 +1754,7 @@ static void LinearPickSeeds( int i; int iLeftSeed = 0; int iRightSeed = 1; - float maxNormalInnerWidth = 0.0; + RtreeDValue maxNormalInnerWidth = (RtreeDValue)0; /* Pick two "seed" cells from the array of cells. The algorithm used ** here is the LinearPickSeeds algorithm from Gutman[1984]. The @@ -1739,18 +1762,18 @@ static void LinearPickSeeds( ** variables iLeftSeek and iRightSeed. */ for(i=0; i<pRtree->nDim; i++){ - float x1 = DCOORD(aCell[0].aCoord[i*2]); - float x2 = DCOORD(aCell[0].aCoord[i*2+1]); - float x3 = x1; - float x4 = x2; + RtreeDValue x1 = DCOORD(aCell[0].aCoord[i*2]); + RtreeDValue x2 = DCOORD(aCell[0].aCoord[i*2+1]); + RtreeDValue x3 = x1; + RtreeDValue x4 = x2; int jj; int iCellLeft = 0; int iCellRight = 0; for(jj=1; jj<nCell; jj++){ - float left = DCOORD(aCell[jj].aCoord[i*2]); - float right = DCOORD(aCell[jj].aCoord[i*2+1]); + RtreeDValue left = DCOORD(aCell[jj].aCoord[i*2]); + RtreeDValue right = DCOORD(aCell[jj].aCoord[i*2+1]); if( left<x1 ) x1 = left; if( right>x4 ) x4 = right; @@ -1765,7 +1788,7 @@ static void LinearPickSeeds( } if( x4!=x1 ){ - float normalwidth = (x3 - x2) / (x4 - x1); + RtreeDValue normalwidth = (x3 - x2) / (x4 - x1); if( normalwidth>maxNormalInnerWidth ){ iLeftSeed = iCellLeft; iRightSeed = iCellRight; @@ -1794,13 +1817,13 @@ static RtreeCell *QuadraticPickNext( #define FABS(a) ((a)<0.0?-1.0*(a):(a)) int iSelect = -1; - float fDiff; + RtreeDValue fDiff; int ii; for(ii=0; ii<nCell; ii++){ if( aiUsed[ii]==0 ){ - float left = cellGrowth(pRtree, pLeftBox, &aCell[ii]); - float right = cellGrowth(pRtree, pLeftBox, &aCell[ii]); - float diff = FABS(right-left); + RtreeDValue left = cellGrowth(pRtree, pLeftBox, &aCell[ii]); + RtreeDValue right = cellGrowth(pRtree, pLeftBox, &aCell[ii]); + RtreeDValue diff = FABS(right-left); if( iSelect<0 || diff>fDiff ){ fDiff = diff; iSelect = ii; @@ -1827,13 +1850,13 @@ static void QuadraticPickSeeds( int iLeftSeed = 0; int iRightSeed = 1; - float fWaste = 0.0; + RtreeDValue fWaste = 0.0; for(ii=0; ii<nCell; ii++){ for(jj=ii+1; jj<nCell; jj++){ - float right = cellArea(pRtree, &aCell[jj]); - float growth = cellGrowth(pRtree, &aCell[ii], &aCell[jj]); - float waste = growth - right; + RtreeDValue right = cellArea(pRtree, &aCell[jj]); + RtreeDValue growth = cellGrowth(pRtree, &aCell[ii], &aCell[jj]); + RtreeDValue waste = growth - right; if( waste>fWaste ){ iLeftSeed = ii; @@ -1868,7 +1891,7 @@ static void QuadraticPickSeeds( static void SortByDistance( int *aIdx, int nIdx, - float *aDistance, + RtreeDValue *aDistance, int *aSpare ){ if( nIdx>1 ){ @@ -1894,8 +1917,8 @@ static void SortByDistance( aIdx[iLeft+iRight] = aLeft[iLeft]; iLeft++; }else{ - float fLeft = aDistance[aLeft[iLeft]]; - float fRight = aDistance[aRight[iRight]]; + RtreeDValue fLeft = aDistance[aLeft[iLeft]]; + RtreeDValue fRight = aDistance[aRight[iRight]]; if( fLeft<fRight ){ aIdx[iLeft+iRight] = aLeft[iLeft]; iLeft++; @@ -1911,8 +1934,8 @@ static void SortByDistance( { int jj; for(jj=1; jj<nIdx; jj++){ - float left = aDistance[aIdx[jj-1]]; - float right = aDistance[aIdx[jj]]; + RtreeDValue left = aDistance[aIdx[jj-1]]; + RtreeDValue right = aDistance[aIdx[jj]]; assert( left<=right ); } } @@ -1955,10 +1978,10 @@ static void SortByDimension( memcpy(aSpare, aLeft, sizeof(int)*nLeft); aLeft = aSpare; while( iLeft<nLeft || iRight<nRight ){ - double xleft1 = DCOORD(aCell[aLeft[iLeft]].aCoord[iDim*2]); - double xleft2 = DCOORD(aCell[aLeft[iLeft]].aCoord[iDim*2+1]); - double xright1 = DCOORD(aCell[aRight[iRight]].aCoord[iDim*2]); - double xright2 = DCOORD(aCell[aRight[iRight]].aCoord[iDim*2+1]); + RtreeDValue xleft1 = DCOORD(aCell[aLeft[iLeft]].aCoord[iDim*2]); + RtreeDValue xleft2 = DCOORD(aCell[aLeft[iLeft]].aCoord[iDim*2+1]); + RtreeDValue xright1 = DCOORD(aCell[aRight[iRight]].aCoord[iDim*2]); + RtreeDValue xright2 = DCOORD(aCell[aRight[iRight]].aCoord[iDim*2+1]); if( (iLeft!=nLeft) && ((iRight==nRight) || (xleft1<xright1) || (xleft1==xright1 && xleft2<xright2) @@ -1976,10 +1999,10 @@ static void SortByDimension( { int jj; for(jj=1; jj<nIdx; jj++){ - float xleft1 = aCell[aIdx[jj-1]].aCoord[iDim*2]; - float xleft2 = aCell[aIdx[jj-1]].aCoord[iDim*2+1]; - float xright1 = aCell[aIdx[jj]].aCoord[iDim*2]; - float xright2 = aCell[aIdx[jj]].aCoord[iDim*2+1]; + RtreeDValue xleft1 = aCell[aIdx[jj-1]].aCoord[iDim*2]; + RtreeDValue xleft2 = aCell[aIdx[jj-1]].aCoord[iDim*2+1]; + RtreeDValue xright1 = aCell[aIdx[jj]].aCoord[iDim*2]; + RtreeDValue xright2 = aCell[aIdx[jj]].aCoord[iDim*2+1]; assert( xleft1<=xright1 && (xleft1<xright1 || xleft2<=xright2) ); } } @@ -2006,7 +2029,7 @@ static int splitNodeStartree( int iBestDim = 0; int iBestSplit = 0; - float fBestMargin = 0.0; + RtreeDValue fBestMargin = 0.0; int nByte = (pRtree->nDim+1)*(sizeof(int*)+nCell*sizeof(int)); @@ -2027,9 +2050,9 @@ static int splitNodeStartree( } for(ii=0; ii<pRtree->nDim; ii++){ - float margin = 0.0; - float fBestOverlap = 0.0; - float fBestArea = 0.0; + RtreeDValue margin = 0.0; + RtreeDValue fBestOverlap = 0.0; + RtreeDValue fBestArea = 0.0; int iBestLeft = 0; int nLeft; @@ -2041,8 +2064,8 @@ static int splitNodeStartree( RtreeCell left; RtreeCell right; int kk; - float overlap; - float area; + RtreeDValue overlap; + RtreeDValue area; memcpy(&left, &aCell[aaSorted[ii][0]], sizeof(RtreeCell)); memcpy(&right, &aCell[aaSorted[ii][nCell-1]], sizeof(RtreeCell)); @@ -2125,7 +2148,7 @@ static int splitNodeGuttman( for(i=nCell-2; i>0; i--){ RtreeCell *pNext; pNext = PickNext(pRtree, aCell, nCell, pBboxLeft, pBboxRight, aiUsed); - float diff = + RtreeDValue diff = cellGrowth(pRtree, pBboxLeft, pNext) - cellGrowth(pRtree, pBboxRight, pNext) ; @@ -2458,32 +2481,34 @@ static int Reinsert( int *aOrder; int *aSpare; RtreeCell *aCell; - float *aDistance; + RtreeDValue *aDistance; int nCell; - float aCenterCoord[RTREE_MAX_DIMENSIONS]; + RtreeDValue aCenterCoord[RTREE_MAX_DIMENSIONS]; int iDim; int ii; int rc = SQLITE_OK; + int n; - memset(aCenterCoord, 0, sizeof(float)*RTREE_MAX_DIMENSIONS); + memset(aCenterCoord, 0, sizeof(RtreeDValue)*RTREE_MAX_DIMENSIONS); nCell = NCELL(pNode)+1; + n = (nCell+1)&(~1); /* Allocate the buffers used by this operation. The allocation is ** relinquished before this function returns. */ - aCell = (RtreeCell *)sqlite3_malloc(nCell * ( - sizeof(RtreeCell) + /* aCell array */ - sizeof(int) + /* aOrder array */ - sizeof(int) + /* aSpare array */ - sizeof(float) /* aDistance array */ + aCell = (RtreeCell *)sqlite3_malloc(n * ( + sizeof(RtreeCell) + /* aCell array */ + sizeof(int) + /* aOrder array */ + sizeof(int) + /* aSpare array */ + sizeof(RtreeDValue) /* aDistance array */ )); if( !aCell ){ return SQLITE_NOMEM; } - aOrder = (int *)&aCell[nCell]; - aSpare = (int *)&aOrder[nCell]; - aDistance = (float *)&aSpare[nCell]; + aOrder = (int *)&aCell[n]; + aSpare = (int *)&aOrder[n]; + aDistance = (RtreeDValue *)&aSpare[n]; for(ii=0; ii<nCell; ii++){ if( ii==(nCell-1) ){ @@ -2493,19 +2518,19 @@ static int Reinsert( } aOrder[ii] = ii; for(iDim=0; iDim<pRtree->nDim; iDim++){ - aCenterCoord[iDim] += (float)DCOORD(aCell[ii].aCoord[iDim*2]); - aCenterCoord[iDim] += (float)DCOORD(aCell[ii].aCoord[iDim*2+1]); + aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2]); + aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2+1]); } } for(iDim=0; iDim<pRtree->nDim; iDim++){ - aCenterCoord[iDim] = (float)(aCenterCoord[iDim]/((float)nCell*2.0)); + aCenterCoord[iDim] = (aCenterCoord[iDim]/(nCell*(RtreeDValue)2)); } for(ii=0; ii<nCell; ii++){ aDistance[ii] = 0.0; for(iDim=0; iDim<pRtree->nDim; iDim++){ - float coord = (float)(DCOORD(aCell[ii].aCoord[iDim*2+1]) - - DCOORD(aCell[ii].aCoord[iDim*2])); + RtreeDValue coord = (DCOORD(aCell[ii].aCoord[iDim*2+1]) - + DCOORD(aCell[ii].aCoord[iDim*2])); aDistance[ii] += (coord-aCenterCoord[iDim])*(coord-aCenterCoord[iDim]); } } @@ -2747,16 +2772,19 @@ static int rtreeUpdate( /* Populate the cell.aCoord[] array. The first coordinate is azData[3]. */ assert( nData==(pRtree->nDim*2 + 3) ); +#ifndef SQLITE_RTREE_INT_ONLY if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ for(ii=0; ii<(pRtree->nDim*2); ii+=2){ - cell.aCoord[ii].f = (float)sqlite3_value_double(azData[ii+3]); - cell.aCoord[ii+1].f = (float)sqlite3_value_double(azData[ii+4]); + cell.aCoord[ii].f = (RtreeValue)sqlite3_value_double(azData[ii+3]); + cell.aCoord[ii+1].f = (RtreeValue)sqlite3_value_double(azData[ii+4]); if( cell.aCoord[ii].f>cell.aCoord[ii+1].f ){ rc = SQLITE_CONSTRAINT; goto constraint; } } - }else{ + }else +#endif + { for(ii=0; ii<(pRtree->nDim*2); ii+=2){ cell.aCoord[ii].i = sqlite3_value_int(azData[ii+3]); cell.aCoord[ii+1].i = sqlite3_value_int(azData[ii+4]); @@ -3154,7 +3182,13 @@ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ sqlite3_snprintf(512-nCell,&zCell[nCell],"%lld", cell.iRowid); nCell = (int)strlen(zCell); for(jj=0; jj<tree.nDim*2; jj++){ - sqlite3_snprintf(512-nCell,&zCell[nCell]," %f",(double)cell.aCoord[jj].f); +#ifndef SQLITE_RTREE_INT_ONLY + sqlite3_snprintf(512-nCell,&zCell[nCell], " %f", + (double)cell.aCoord[jj].f); +#else + sqlite3_snprintf(512-nCell,&zCell[nCell], " %d", + cell.aCoord[jj].i); +#endif nCell = (int)strlen(zCell); } @@ -3196,7 +3230,11 @@ int sqlite3RtreeInit(sqlite3 *db){ rc = sqlite3_create_function(db, "rtreedepth", 1, utf8, 0,rtreedepth, 0, 0); } if( rc==SQLITE_OK ){ +#ifdef SQLITE_RTREE_INT_ONLY + void *c = (void *)RTREE_COORD_INT32; +#else void *c = (void *)RTREE_COORD_REAL32; +#endif rc = sqlite3_create_module_v2(db, "rtree", &rtreeModule, c, 0); } if( rc==SQLITE_OK ){ @@ -3230,7 +3268,7 @@ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ RtreeMatchArg *pBlob; int nBlob; - nBlob = sizeof(RtreeMatchArg) + (nArg-1)*sizeof(double); + nBlob = sizeof(RtreeMatchArg) + (nArg-1)*sizeof(RtreeDValue); pBlob = (RtreeMatchArg *)sqlite3_malloc(nBlob); if( !pBlob ){ sqlite3_result_error_nomem(ctx); @@ -3241,7 +3279,11 @@ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ pBlob->pContext = pGeomCtx->pContext; pBlob->nParam = nArg; for(i=0; i<nArg; i++){ +#ifdef SQLITE_RTREE_INT_ONLY + pBlob->aParam[i] = sqlite3_value_int64(aArg[i]); +#else pBlob->aParam[i] = sqlite3_value_double(aArg[i]); +#endif } sqlite3_result_blob(ctx, pBlob, nBlob, doSqlite3Free); } @@ -3253,7 +3295,7 @@ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ int sqlite3_rtree_geometry_callback( sqlite3 *db, const char *zGeom, - int (*xGeom)(sqlite3_rtree_geometry *, int, double *, int *), + int (*xGeom)(sqlite3_rtree_geometry *, int, RtreeDValue *, int *), void *pContext ){ RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */ diff --git a/lib/libsqlite3/ext/rtree/rtree1.test b/lib/libsqlite3/ext/rtree/rtree1.test index 583b0285076..e3c7d68e82d 100644 --- a/lib/libsqlite3/ext/rtree/rtree1.test +++ b/lib/libsqlite3/ext/rtree/rtree1.test @@ -104,6 +104,18 @@ for {set nCol 1} {$nCol<[llength $cols]} {incr nCol} { catchsql { DROP TABLE t1 } } +# Like execsql except display output as integer where that can be +# done without loss of information. +# +proc execsql_intout {sql} { + set out {} + foreach term [execsql $sql] { + regsub {\.0$} $term {} term + lappend out $term + } + return $out +} + # Test that it is possible to open an existing database that contains # r-tree tables. # @@ -117,8 +129,8 @@ do_test rtree-1.4.1 { do_test rtree-1.4.2 { db close sqlite3 db test.db - execsql { SELECT * FROM t1 ORDER BY ii } -} {1 5.0 10.0 2 15.0 20.0} + execsql_intout { SELECT * FROM t1 ORDER BY ii } +} {1 5 10 2 15 20} do_test rtree-1.4.3 { execsql { DROP TABLE t1 } } {} @@ -127,12 +139,12 @@ do_test rtree-1.4.3 { # column names. # do_test rtree-1.5.1 { - execsql { + execsql_intout { CREATE VIRTUAL TABLE t1 USING rtree("the key", "x dim.", "x2'dim"); INSERT INTO t1 VALUES(1, 2, 3); SELECT "the key", "x dim.", "x2'dim" FROM t1; } -} {1 2.0 3.0} +} {1 2 3} do_test rtree-1.5.1 { execsql { DROP TABLE t1 } } {} @@ -161,8 +173,8 @@ do_test rtree-2.1.1 { do_test rtree-2.1.2 { execsql { INSERT INTO t1 VALUES(NULL, 1, 3, 2, 4) } - execsql { SELECT * FROM t1 } -} {1 1.0 3.0 2.0 4.0} + execsql_intout { SELECT * FROM t1 } +} {1 1 3 2 4} do_test rtree-2.1.3 { execsql { INSERT INTO t1 VALUES(NULL, 1, 3, 2, 4) } execsql { SELECT rowid FROM t1 ORDER BY rowid } @@ -201,17 +213,17 @@ do_test rtree-3.1.1 { } } {} do_test rtree-3.1.2 { - execsql { + execsql_intout { INSERT INTO t1 VALUES(5, 1, 3, 2, 4); SELECT * FROM t1; } -} {5 1.0 3.0 2.0 4.0} +} {5 1 3 2 4} do_test rtree-3.1.3 { - execsql { + execsql_intout { INSERT INTO t1 VALUES(6, 2, 6, 4, 8); SELECT * FROM t1; } -} {5 1.0 3.0 2.0 4.0 6 2.0 6.0 4.0 8.0} +} {5 1 3 2 4 6 2 6 4 8} # Test the constraint on the coordinates (c[i]<=c[i+1] where (i%2==0)): do_test rtree-3.2.1 { @@ -228,25 +240,25 @@ do_test rtree-5.1.1 { execsql { CREATE VIRTUAL TABLE t2 USING rtree(ii, x1, x2) } } {} do_test rtree-5.1.2 { - execsql { + execsql_intout { INSERT INTO t2 VALUES(1, 10, 20); INSERT INTO t2 VALUES(2, 30, 40); INSERT INTO t2 VALUES(3, 50, 60); SELECT * FROM t2 ORDER BY ii; } -} {1 10.0 20.0 2 30.0 40.0 3 50.0 60.0} +} {1 10 20 2 30 40 3 50 60} do_test rtree-5.1.3 { - execsql { + execsql_intout { DELETE FROM t2 WHERE ii=2; SELECT * FROM t2 ORDER BY ii; } -} {1 10.0 20.0 3 50.0 60.0} +} {1 10 20 3 50 60} do_test rtree-5.1.4 { - execsql { + execsql_intout { DELETE FROM t2 WHERE ii=1; SELECT * FROM t2 ORDER BY ii; } -} {3 50.0 60.0} +} {3 50 60} do_test rtree-5.1.5 { execsql { DELETE FROM t2 WHERE ii=3; @@ -264,16 +276,16 @@ do_test rtree-6.1.1 { execsql { CREATE VIRTUAL TABLE t3 USING rtree(ii, x1, x2, y1, y2) } } {} do_test rtree-6.1.2 { - execsql { + execsql_intout { INSERT INTO t3 VALUES(1, 2, 3, 4, 5); UPDATE t3 SET x2=5; SELECT * FROM t3; } -} {1 2.0 5.0 4.0 5.0} +} {1 2 5 4 5} do_test rtree-6.1.3 { execsql { UPDATE t3 SET ii = 2 } - execsql { SELECT * FROM t3 } -} {2 2.0 5.0 4.0 5.0} + execsql_intout { SELECT * FROM t3 } +} {2 2 5 4 5} #---------------------------------------------------------------------------- # Test cases rtree-7.* test rename operations. @@ -286,29 +298,29 @@ do_test rtree-7.1.1 { } {} do_test rtree-7.1.2 { execsql { ALTER TABLE t4 RENAME TO t5 } - execsql { SELECT * FROM t5 } -} {1 2.0 3.0 4.0 5.0 6.0 7.0} + execsql_intout { SELECT * FROM t5 } +} {1 2 3 4 5 6 7} do_test rtree-7.1.3 { db close sqlite3 db test.db - execsql { SELECT * FROM t5 } -} {1 2.0 3.0 4.0 5.0 6.0 7.0} + execsql_intout { SELECT * FROM t5 } +} {1 2 3 4 5 6 7} do_test rtree-7.1.4 { execsql { ALTER TABLE t5 RENAME TO 'raisara "one"'''} - execsql { SELECT * FROM "raisara ""one""'" } -} {1 2.0 3.0 4.0 5.0 6.0 7.0} + execsql_intout { SELECT * FROM "raisara ""one""'" } +} {1 2 3 4 5 6 7} do_test rtree-7.1.5 { - execsql { SELECT * FROM 'raisara "one"''' } -} {1 2.0 3.0 4.0 5.0 6.0 7.0} + execsql_intout { SELECT * FROM 'raisara "one"''' } +} {1 2 3 4 5 6 7} do_test rtree-7.1.6 { execsql { ALTER TABLE "raisara ""one""'" RENAME TO "abc 123" } - execsql { SELECT * FROM "abc 123" } -} {1 2.0 3.0 4.0 5.0 6.0 7.0} + execsql_intout { SELECT * FROM "abc 123" } +} {1 2 3 4 5 6 7} do_test rtree-7.1.7 { db close sqlite3 db test.db - execsql { SELECT * FROM "abc 123" } -} {1 2.0 3.0 4.0 5.0 6.0 7.0} + execsql_intout { SELECT * FROM "abc 123" } +} {1 2 3 4 5 6 7} # An error midway through a rename operation. do_test rtree-7.2.1 { @@ -318,8 +330,8 @@ do_test rtree-7.2.1 { catchsql { ALTER TABLE "abc 123" RENAME TO t4 } } {1 {SQL logic error or missing database}} do_test rtree-7.2.2 { - execsql { SELECT * FROM "abc 123" } -} {1 2.0 3.0 4.0 5.0 6.0 7.0} + execsql_intout { SELECT * FROM "abc 123" } +} {1 2 3 4 5 6 7} do_test rtree-7.2.3 { execsql { DROP TABLE t4_node; @@ -330,13 +342,13 @@ do_test rtree-7.2.3 { do_test rtree-7.2.4 { db close sqlite3 db test.db - execsql { SELECT * FROM "abc 123" } -} {1 2.0 3.0 4.0 5.0 6.0 7.0} + execsql_intout { SELECT * FROM "abc 123" } +} {1 2 3 4 5 6 7} do_test rtree-7.2.5 { execsql { DROP TABLE t4_rowid } execsql { ALTER TABLE "abc 123" RENAME TO t4 } - execsql { SELECT * FROM t4 } -} {1 2.0 3.0 4.0 5.0 6.0 7.0} + execsql_intout { SELECT * FROM t4 } +} {1 2 3 4 5 6 7} #---------------------------------------------------------------------------- diff --git a/lib/libsqlite3/ext/rtree/rtree4.test b/lib/libsqlite3/ext/rtree/rtree4.test index 708d335b067..a3872b07356 100644 --- a/lib/libsqlite3/ext/rtree/rtree4.test +++ b/lib/libsqlite3/ext/rtree/rtree4.test @@ -27,21 +27,38 @@ if {[info exists G(isquick)] && $G(isquick)} { set ::NROW 250 } -# Return a floating point number between -X and X. -# -proc rand {X} { - return [expr {int((rand()-0.5)*1024.0*$X)/512.0}] -} - -# Return a positive floating point number less than or equal to X -# -proc randincr {X} { - while 1 { - set r [expr {int(rand()*$X*32.0)/32.0}] - if {$r>0.0} {return $r} +ifcapable !rtree_int_only { + # Return a floating point number between -X and X. + # + proc rand {X} { + return [expr {int((rand()-0.5)*1024.0*$X)/512.0}] + } + + # Return a positive floating point number less than or equal to X + # + proc randincr {X} { + while 1 { + set r [expr {int(rand()*$X*32.0)/32.0}] + if {$r>0.0} {return $r} + } + } +} else { + # For rtree_int_only, return an number between -X and X. + # + proc rand {X} { + return [expr {int((rand()-0.5)*2*$X)}] + } + + # Return a positive integer less than or equal to X + # + proc randincr {X} { + while 1 { + set r [expr {int(rand()*$X)+1}] + if {$r>0} {return $r} + } } } - + # Scramble the $inlist into a random order. # proc scramble {inlist} { diff --git a/lib/libsqlite3/ext/rtree/rtree5.test b/lib/libsqlite3/ext/rtree/rtree5.test index ea2946f918f..8990772356a 100644 --- a/lib/libsqlite3/ext/rtree/rtree5.test +++ b/lib/libsqlite3/ext/rtree/rtree5.test @@ -49,9 +49,11 @@ do_test rtree5-1.6 { do_test rtree5-1.7 { execsql { SELECT count(*) FROM t1 WHERE x1==5 } } {1} -do_test rtree5-1.8 { - execsql { SELECT count(*) FROM t1 WHERE x1==5.2 } -} {0} +ifcapable !rtree_int_only { + do_test rtree5-1.8 { + execsql { SELECT count(*) FROM t1 WHERE x1==5.2 } + } {0} +} do_test rtree5-1.9 { execsql { SELECT count(*) FROM t1 WHERE x1==5.0 } } {1} diff --git a/lib/libsqlite3/ext/rtree/rtree6.test b/lib/libsqlite3/ext/rtree/rtree6.test index ba0e53c9948..92edc8d104c 100644 --- a/lib/libsqlite3/ext/rtree/rtree6.test +++ b/lib/libsqlite3/ext/rtree/rtree6.test @@ -16,7 +16,7 @@ if {![info exists testdir]} { } source $testdir/tester.tcl -ifcapable !rtree { +ifcapable {!rtree || rtree_int_only} { finish_test return } diff --git a/lib/libsqlite3/ext/rtree/rtree7.test b/lib/libsqlite3/ext/rtree/rtree7.test index 31dae0cd8a2..4eee4c219a6 100644 --- a/lib/libsqlite3/ext/rtree/rtree7.test +++ b/lib/libsqlite3/ext/rtree/rtree7.test @@ -24,6 +24,18 @@ ifcapable !rtree||!vacuum { return } +# Like execsql except display output as integer where that can be +# done without loss of information. +# +proc execsql_intout {sql} { + set out {} + foreach term [execsql $sql] { + regsub {\.0$} $term {} term + lappend out $term + } + return $out +} + do_test rtree7-1.1 { execsql { PRAGMA page_size = 1024; @@ -32,27 +44,27 @@ do_test rtree7-1.1 { } } {} do_test rtree7-1.2 { - execsql { SELECT * FROM rt } -} {1 1.0 2.0 3.0 4.0} + execsql_intout { SELECT * FROM rt } +} {1 1 2 3 4} do_test rtree7-1.3 { - execsql { + execsql_intout { PRAGMA page_size = 2048; VACUUM; SELECT * FROM rt; } -} {1 1.0 2.0 3.0 4.0} +} {1 1 2 3 4} do_test rtree7-1.4 { for {set i 2} {$i <= 51} {incr i} { execsql { INSERT INTO rt VALUES($i, 1, 2, 3, 4) } } - execsql { SELECT sum(x1), sum(x2), sum(y1), sum(y2) FROM rt } -} {51.0 102.0 153.0 204.0} + execsql_intout { SELECT sum(x1), sum(x2), sum(y1), sum(y2) FROM rt } +} {51 102 153 204} do_test rtree7-1.5 { - execsql { + execsql_intout { PRAGMA page_size = 512; VACUUM; SELECT sum(x1), sum(x2), sum(y1), sum(y2) FROM rt } -} {51.0 102.0 153.0 204.0} +} {51 102 153 204} finish_test diff --git a/lib/libsqlite3/ext/rtree/rtree9.test b/lib/libsqlite3/ext/rtree/rtree9.test index ddee277ef52..6479516bed6 100644 --- a/lib/libsqlite3/ext/rtree/rtree9.test +++ b/lib/libsqlite3/ext/rtree/rtree9.test @@ -17,6 +17,7 @@ if {![info exists testdir]} { } source $testdir/tester.tcl ifcapable !rtree { finish_test ; return } +ifcapable rtree_int_only { finish_test; return } register_cube_geom db diff --git a/lib/libsqlite3/ext/rtree/rtreeB.test b/lib/libsqlite3/ext/rtree/rtreeB.test index 2756fceedb9..7cb445cc4fa 100644 --- a/lib/libsqlite3/ext/rtree/rtreeB.test +++ b/lib/libsqlite3/ext/rtree/rtreeB.test @@ -18,17 +18,30 @@ if {![info exists testdir]} { source $testdir/tester.tcl ifcapable !rtree { finish_test ; return } -do_test rtreeB-1.1 { - db eval { - CREATE VIRTUAL TABLE t1 USING rtree(ii, x0, y0, x1, y1); - INSERT INTO t1 VALUES(1073741824, 0.0, 0.0, 100.0, 100.0); - INSERT INTO t1 VALUES(2147483646, 0.0, 0.0, 200.0, 200.0); - INSERT INTO t1 VALUES(4294967296, 0.0, 0.0, 300.0, 300.0); - INSERT INTO t1 VALUES(8589934592, 20.0, 20.0, 150.0, 150.0); - INSERT INTO t1 VALUES(9223372036854775807, 150, 150, 400, 400); - SELECT rtreenode(2, data) FROM t1_node; - } -} {{{1073741824 0.000000 0.000000 100.000000 100.000000} {2147483646 0.000000 0.000000 200.000000 200.000000} {4294967296 0.000000 0.000000 300.000000 300.000000} {8589934592 20.000000 20.000000 150.000000 150.000000} {9223372036854775807 150.000000 150.000000 400.000000 400.000000}}} - +ifcapable rtree_int_only { + do_test rtreeB-1.1-intonly { + db eval { + CREATE VIRTUAL TABLE t1 USING rtree(ii, x0, y0, x1, y1); + INSERT INTO t1 VALUES(1073741824, 0.0, 0.0, 100.0, 100.0); + INSERT INTO t1 VALUES(2147483646, 0.0, 0.0, 200.0, 200.0); + INSERT INTO t1 VALUES(4294967296, 0.0, 0.0, 300.0, 300.0); + INSERT INTO t1 VALUES(8589934592, 20.0, 20.0, 150.0, 150.0); + INSERT INTO t1 VALUES(9223372036854775807, 150, 150, 400, 400); + SELECT rtreenode(2, data) FROM t1_node; + } + } {{{1073741824 0 0 100 100} {2147483646 0 0 200 200} {4294967296 0 0 300 300} {8589934592 20 20 150 150} {9223372036854775807 150 150 400 400}}} +} else { + do_test rtreeB-1.1 { + db eval { + CREATE VIRTUAL TABLE t1 USING rtree(ii, x0, y0, x1, y1); + INSERT INTO t1 VALUES(1073741824, 0.0, 0.0, 100.0, 100.0); + INSERT INTO t1 VALUES(2147483646, 0.0, 0.0, 200.0, 200.0); + INSERT INTO t1 VALUES(4294967296, 0.0, 0.0, 300.0, 300.0); + INSERT INTO t1 VALUES(8589934592, 20.0, 20.0, 150.0, 150.0); + INSERT INTO t1 VALUES(9223372036854775807, 150, 150, 400, 400); + SELECT rtreenode(2, data) FROM t1_node; + } + } {{{1073741824 0.000000 0.000000 100.000000 100.000000} {2147483646 0.000000 0.000000 200.000000 200.000000} {4294967296 0.000000 0.000000 300.000000 300.000000} {8589934592 20.000000 20.000000 150.000000 150.000000} {9223372036854775807 150.000000 150.000000 400.000000 400.000000}}} +} finish_test diff --git a/lib/libsqlite3/ext/rtree/sqlite3rtree.h b/lib/libsqlite3/ext/rtree/sqlite3rtree.h index cffb300092e..c849091f29a 100644 --- a/lib/libsqlite3/ext/rtree/sqlite3rtree.h +++ b/lib/libsqlite3/ext/rtree/sqlite3rtree.h @@ -31,7 +31,11 @@ typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry; int sqlite3_rtree_geometry_callback( sqlite3 *db, const char *zGeom, - int (*xGeom)(sqlite3_rtree_geometry *, int nCoord, double *aCoord, int *pRes), +#ifdef SQLITE_RTREE_INT_ONLY + int (*xGeom)(sqlite3_rtree_geometry*, int n, sqlite3_int64 *a, int *pRes), +#else + int (*xGeom)(sqlite3_rtree_geometry*, int n, double *a, int *pRes), +#endif void *pContext ); diff --git a/lib/libsqlite3/src/btree.c b/lib/libsqlite3/src/btree.c index 68345144a8d..28765269250 100644 --- a/lib/libsqlite3/src/btree.c +++ b/lib/libsqlite3/src/btree.c @@ -6789,13 +6789,6 @@ int sqlite3BtreeInsert( ** blob of associated data. */ assert( (pKey==0)==(pCur->pKeyInfo==0) ); - /* If this is an insert into a table b-tree, invalidate any incrblob - ** cursors open on the row being replaced (assuming this is a replace - ** operation - if it is not, the following is a no-op). */ - if( pCur->pKeyInfo==0 ){ - invalidateIncrblobCursors(p, nKey, 0); - } - /* Save the positions of any other cursors open on this table. ** ** In some cases, the call to btreeMoveto() below is a no-op. For @@ -6809,6 +6802,14 @@ int sqlite3BtreeInsert( */ rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); if( rc ) return rc; + + /* If this is an insert into a table b-tree, invalidate any incrblob + ** cursors open on the row being replaced (assuming this is a replace + ** operation - if it is not, the following is a no-op). */ + if( pCur->pKeyInfo==0 ){ + invalidateIncrblobCursors(p, nKey, 0); + } + if( !loc ){ rc = btreeMoveto(pCur, pKey, nKey, appendBias, &loc); if( rc ) return rc; @@ -6919,12 +6920,6 @@ int sqlite3BtreeDelete(BtCursor *pCur){ return SQLITE_ERROR; /* Something has gone awry. */ } - /* If this is a delete operation to remove a row from a table b-tree, - ** invalidate any incrblob cursors open on the row being deleted. */ - if( pCur->pKeyInfo==0 ){ - invalidateIncrblobCursors(p, pCur->info.nKey, 0); - } - iCellDepth = pCur->iPage; iCellIdx = pCur->aiIdx[iCellDepth]; pPage = pCur->apPage[iCellDepth]; @@ -6950,6 +6945,13 @@ int sqlite3BtreeDelete(BtCursor *pCur){ */ rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); if( rc ) return rc; + + /* If this is a delete operation to remove a row from a table b-tree, + ** invalidate any incrblob cursors open on the row being deleted. */ + if( pCur->pKeyInfo==0 ){ + invalidateIncrblobCursors(p, pCur->info.nKey, 0); + } + rc = sqlite3PagerWrite(pPage->pDbPage); if( rc ) return rc; rc = clearCell(pPage, pCell); @@ -7231,13 +7233,13 @@ int sqlite3BtreeClearTable(Btree *p, int iTable, int *pnChange){ sqlite3BtreeEnter(p); assert( p->inTrans==TRANS_WRITE ); - /* Invalidate all incrblob cursors open on table iTable (assuming iTable - ** is the root of a table b-tree - if it is not, the following call is - ** a no-op). */ - invalidateIncrblobCursors(p, 0, 1); - rc = saveAllCursors(pBt, (Pgno)iTable, 0); + if( SQLITE_OK==rc ){ + /* Invalidate all incrblob cursors open on table iTable (assuming iTable + ** is the root of a table b-tree - if it is not, the following call is + ** a no-op). */ + invalidateIncrblobCursors(p, 0, 1); rc = clearDatabasePage(pBt, (Pgno)iTable, 0, pnChange); } sqlite3BtreeLeave(p); @@ -7552,6 +7554,25 @@ static void checkAppendMsg( #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ #ifndef SQLITE_OMIT_INTEGRITY_CHECK + +/* +** Return non-zero if the bit in the IntegrityCk.aPgRef[] array that +** corresponds to page iPg is already set. +*/ +static int getPageReferenced(IntegrityCk *pCheck, Pgno iPg){ + assert( iPg<=pCheck->nPage && sizeof(pCheck->aPgRef[0])==1 ); + return (pCheck->aPgRef[iPg/8] & (1 << (iPg & 0x07))); +} + +/* +** Set the bit in the IntegrityCk.aPgRef[] array that corresponds to page iPg. +*/ +static void setPageReferenced(IntegrityCk *pCheck, Pgno iPg){ + assert( iPg<=pCheck->nPage && sizeof(pCheck->aPgRef[0])==1 ); + pCheck->aPgRef[iPg/8] |= (1 << (iPg & 0x07)); +} + + /* ** Add 1 to the reference count for page iPage. If this is the second ** reference to the page, add an error message to pCheck->zErrMsg. @@ -7566,11 +7587,12 @@ static int checkRef(IntegrityCk *pCheck, Pgno iPage, char *zContext){ checkAppendMsg(pCheck, zContext, "invalid page number %d", iPage); return 1; } - if( pCheck->anRef[iPage]==1 ){ + if( getPageReferenced(pCheck, iPage) ){ checkAppendMsg(pCheck, zContext, "2nd reference to page %d", iPage); return 1; } - return (pCheck->anRef[iPage]++)>1; + setPageReferenced(pCheck, iPage); + return 0; } #ifndef SQLITE_OMIT_AUTOVACUUM @@ -7946,17 +7968,15 @@ char *sqlite3BtreeIntegrityCheck( sqlite3BtreeLeave(p); return 0; } - sCheck.anRef = sqlite3Malloc( (sCheck.nPage+1)*sizeof(sCheck.anRef[0]) ); - if( !sCheck.anRef ){ + + sCheck.aPgRef = sqlite3MallocZero((sCheck.nPage / 8)+ 1); + if( !sCheck.aPgRef ){ *pnErr = 1; sqlite3BtreeLeave(p); return 0; } - for(i=0; i<=sCheck.nPage; i++){ sCheck.anRef[i] = 0; } i = PENDING_BYTE_PAGE(pBt); - if( i<=sCheck.nPage ){ - sCheck.anRef[i] = 1; - } + if( i<=sCheck.nPage ) setPageReferenced(&sCheck, i); sqlite3StrAccumInit(&sCheck.errMsg, zErr, sizeof(zErr), 20000); sCheck.errMsg.useMalloc = 2; @@ -7981,18 +8001,18 @@ char *sqlite3BtreeIntegrityCheck( */ for(i=1; i<=sCheck.nPage && sCheck.mxErr; i++){ #ifdef SQLITE_OMIT_AUTOVACUUM - if( sCheck.anRef[i]==0 ){ + if( getPageReferenced(&sCheck, i)==0 ){ checkAppendMsg(&sCheck, 0, "Page %d is never used", i); } #else /* If the database supports auto-vacuum, make sure no tables contain ** references to pointer-map pages. */ - if( sCheck.anRef[i]==0 && + if( getPageReferenced(&sCheck, i)==0 && (PTRMAP_PAGENO(pBt, i)!=i || !pBt->autoVacuum) ){ checkAppendMsg(&sCheck, 0, "Page %d is never used", i); } - if( sCheck.anRef[i]!=0 && + if( getPageReferenced(&sCheck, i)!=0 && (PTRMAP_PAGENO(pBt, i)==i && pBt->autoVacuum) ){ checkAppendMsg(&sCheck, 0, "Pointer map page %d is referenced", i); } @@ -8013,7 +8033,7 @@ char *sqlite3BtreeIntegrityCheck( /* Clean up and report errors. */ sqlite3BtreeLeave(p); - sqlite3_free(sCheck.anRef); + sqlite3_free(sCheck.aPgRef); if( sCheck.mallocFailed ){ sqlite3StrAccumReset(&sCheck.errMsg); *pnErr = sCheck.nErr+1; diff --git a/lib/libsqlite3/src/btreeInt.h b/lib/libsqlite3/src/btreeInt.h index 841e2c6eab4..0d21497966a 100644 --- a/lib/libsqlite3/src/btreeInt.h +++ b/lib/libsqlite3/src/btreeInt.h @@ -631,12 +631,18 @@ struct BtCursor { /* ** This structure is passed around through all the sanity checking routines ** in order to keep track of some global state information. +** +** The aRef[] array is allocated so that there is 1 bit for each page in +** the database. As the integrity-check proceeds, for each page used in +** the database the corresponding bit is set. This allows integrity-check to +** detect pages that are used twice and orphaned pages (both of which +** indicate corruption). */ typedef struct IntegrityCk IntegrityCk; struct IntegrityCk { BtShared *pBt; /* The tree being checked out */ Pager *pPager; /* The associated pager. Also accessible by pBt->pPager */ - int *anRef; /* Number of times each page is referenced */ + u8 *aPgRef; /* 1 bit per page in the db (see above) */ Pgno nPage; /* Number of pages in the database */ int mxErr; /* Stop accumulating errors when this reaches zero */ int nErr; /* Number of messages written to zErrMsg so far */ diff --git a/lib/libsqlite3/src/build.c b/lib/libsqlite3/src/build.c index daa54304060..6fc59dade40 100644 --- a/lib/libsqlite3/src/build.c +++ b/lib/libsqlite3/src/build.c @@ -537,7 +537,7 @@ void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ sqlite3DbFree(db, pTable->zColAff); sqlite3SelectDelete(db, pTable->pSelect); #ifndef SQLITE_OMIT_CHECK - sqlite3ExprDelete(db, pTable->pCheck); + sqlite3ExprListDelete(db, pTable->pCheck); #endif #ifndef SQLITE_OMIT_VIRTUALTABLE sqlite3VtabClear(db, pTable); @@ -1200,15 +1200,17 @@ void sqlite3AddCheckConstraint( Parse *pParse, /* Parsing context */ Expr *pCheckExpr /* The check expression */ ){ - sqlite3 *db = pParse->db; #ifndef SQLITE_OMIT_CHECK Table *pTab = pParse->pNewTable; if( pTab && !IN_DECLARE_VTAB ){ - pTab->pCheck = sqlite3ExprAnd(db, pTab->pCheck, pCheckExpr); + pTab->pCheck = sqlite3ExprListAppend(pParse, pTab->pCheck, pCheckExpr); + if( pParse->constraintName.n ){ + sqlite3ExprListSetName(pParse, pTab->pCheck, &pParse->constraintName, 1); + } }else #endif { - sqlite3ExprDelete(db, pCheckExpr); + sqlite3ExprDelete(pParse->db, pCheckExpr); } } @@ -1478,6 +1480,8 @@ void sqlite3EndTable( if( p->pCheck ){ SrcList sSrc; /* Fake SrcList for pParse->pNewTable */ NameContext sNC; /* Name context for pParse->pNewTable */ + ExprList *pList; /* List of all CHECK constraints */ + int i; /* Loop counter */ memset(&sNC, 0, sizeof(sNC)); memset(&sSrc, 0, sizeof(sSrc)); @@ -1488,8 +1492,11 @@ void sqlite3EndTable( sNC.pParse = pParse; sNC.pSrcList = &sSrc; sNC.isCheck = 1; - if( sqlite3ResolveExprNames(&sNC, p->pCheck) ){ - return; + pList = p->pCheck; + for(i=0; i<pList->nExpr; i++){ + if( sqlite3ResolveExprNames(&sNC, pList->a[i].pExpr) ){ + return; + } } } #endif /* !defined(SQLITE_OMIT_CHECK) */ @@ -3040,19 +3047,21 @@ exit_drop_index: } /* -** pArray is a pointer to an array of objects. Each object in the -** array is szEntry bytes in size. This routine allocates a new -** object on the end of the array. +** pArray is a pointer to an array of objects. Each object in the +** array is szEntry bytes in size. This routine uses sqlite3DbRealloc() +** to extend the array so that there is space for a new object at the end. ** -** *pnEntry is the number of entries already in use. *pnAlloc is -** the previously allocated size of the array. initSize is the -** suggested initial array size allocation. +** When this function is called, *pnEntry contains the current size of +** the array (in entries - so the allocation is ((*pnEntry) * szEntry) bytes +** in total). ** -** The index of the new entry is returned in *pIdx. +** If the realloc() is successful (i.e. if no OOM condition occurs), the +** space allocated for the new object is zeroed, *pnEntry updated to +** reflect the new size of the array and a pointer to the new allocation +** returned. *pIdx is set to the index of the new array entry in this case. ** -** This routine returns a pointer to the array of objects. This -** might be the same as the pArray parameter or it might be a different -** pointer if the array was resized. +** Otherwise, if the realloc() fails, *pIdx is set to -1, *pnEntry remains +** unchanged and a copy of pArray returned. */ void *sqlite3ArrayAllocate( sqlite3 *db, /* Connection to notify of malloc failures */ diff --git a/lib/libsqlite3/src/callback.c b/lib/libsqlite3/src/callback.c index ce849085c22..a515e05e2a0 100644 --- a/lib/libsqlite3/src/callback.c +++ b/lib/libsqlite3/src/callback.c @@ -223,38 +223,57 @@ CollSeq *sqlite3FindCollSeq( ** that uses encoding enc. The value returned indicates how well the ** request is matched. A higher value indicates a better match. ** +** If nArg is -1 that means to only return a match (non-zero) if p->nArg +** is also -1. In other words, we are searching for a function that +** takes a variable number of arguments. +** +** If nArg is -2 that means that we are searching for any function +** regardless of the number of arguments it uses, so return a positive +** match score for any +** ** The returned value is always between 0 and 6, as follows: ** -** 0: Not a match, or if nArg<0 and the function is has no implementation. -** 1: A variable arguments function that prefers UTF-8 when a UTF-16 -** encoding is requested, or vice versa. -** 2: A variable arguments function that uses UTF-16BE when UTF-16LE is -** requested, or vice versa. -** 3: A variable arguments function using the same text encoding. -** 4: A function with the exact number of arguments requested that -** prefers UTF-8 when a UTF-16 encoding is requested, or vice versa. -** 5: A function with the exact number of arguments requested that -** prefers UTF-16LE when UTF-16BE is requested, or vice versa. -** 6: An exact match. +** 0: Not a match. +** 1: UTF8/16 conversion required and function takes any number of arguments. +** 2: UTF16 byte order change required and function takes any number of args. +** 3: encoding matches and function takes any number of arguments +** 4: UTF8/16 conversion required - argument count matches exactly +** 5: UTF16 byte order conversion required - argument count matches exactly +** 6: Perfect match: encoding and argument count match exactly. ** +** If nArg==(-2) then any function with a non-null xStep or xFunc is +** a perfect match and any function with both xStep and xFunc NULL is +** a non-match. */ -static int matchQuality(FuncDef *p, int nArg, u8 enc){ - int match = 0; - if( p->nArg==-1 || p->nArg==nArg - || (nArg==-1 && (p->xFunc!=0 || p->xStep!=0)) - ){ +#define FUNC_PERFECT_MATCH 6 /* The score for a perfect match */ +static int matchQuality( + FuncDef *p, /* The function we are evaluating for match quality */ + int nArg, /* Desired number of arguments. (-1)==any */ + u8 enc /* Desired text encoding */ +){ + int match; + + /* nArg of -2 is a special case */ + if( nArg==(-2) ) return (p->xFunc==0 && p->xStep==0) ? 0 : FUNC_PERFECT_MATCH; + + /* Wrong number of arguments means "no match" */ + if( p->nArg!=nArg && p->nArg>=0 ) return 0; + + /* Give a better score to a function with a specific number of arguments + ** than to function that accepts any number of arguments. */ + if( p->nArg==nArg ){ + match = 4; + }else{ match = 1; - if( p->nArg==nArg || nArg==-1 ){ - match = 4; - } - if( enc==p->iPrefEnc ){ - match += 2; - } - else if( (enc==SQLITE_UTF16LE && p->iPrefEnc==SQLITE_UTF16BE) || - (enc==SQLITE_UTF16BE && p->iPrefEnc==SQLITE_UTF16LE) ){ - match += 1; - } } + + /* Bonus points if the text encoding matches */ + if( enc==p->iPrefEnc ){ + match += 2; /* Exact encoding match */ + }else if( (enc & p->iPrefEnc & 2)!=0 ){ + match += 1; /* Both are UTF16, but with different byte orders */ + } + return match; } @@ -310,13 +329,12 @@ void sqlite3FuncDefInsert( ** ** If the createFlag argument is true, then a new (blank) FuncDef ** structure is created and liked into the "db" structure if a -** no matching function previously existed. When createFlag is true -** and the nArg parameter is -1, then only a function that accepts -** any number of arguments will be returned. +** no matching function previously existed. ** -** If createFlag is false and nArg is -1, then the first valid -** function found is returned. A function is valid if either xFunc -** or xStep is non-zero. +** If nArg is -2, then the first valid function found is returned. A +** function is valid if either xFunc or xStep is non-zero. The nArg==(-2) +** case is used to see if zName is a valid function name for some number +** of arguments. If nArg is -2, then createFlag must be 0. ** ** If createFlag is false, then a function with the required name and ** number of arguments may be returned even if the eTextRep flag does not @@ -328,14 +346,15 @@ FuncDef *sqlite3FindFunction( int nName, /* Number of characters in the name */ int nArg, /* Number of arguments. -1 means any number */ u8 enc, /* Preferred text encoding */ - int createFlag /* Create new entry if true and does not otherwise exist */ + u8 createFlag /* Create new entry if true and does not otherwise exist */ ){ FuncDef *p; /* Iterator variable */ FuncDef *pBest = 0; /* Best match found so far */ int bestScore = 0; /* Score of best match */ int h; /* Hash value */ - + assert( nArg>=(-2) ); + assert( nArg>=(-1) || createFlag==0 ); assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); h = (sqlite3UpperToLower[(u8)zName[0]] + nName) % ArraySize(db->aFunc.a); @@ -381,7 +400,7 @@ FuncDef *sqlite3FindFunction( ** exact match for the name, number of arguments and encoding, then add a ** new entry to the hash table and return it. */ - if( createFlag && (bestScore<6 || pBest->nArg!=nArg) && + if( createFlag && bestScore<FUNC_PERFECT_MATCH && (pBest = sqlite3DbMallocZero(db, sizeof(*pBest)+nName+1))!=0 ){ pBest->zName = (char *)&pBest[1]; pBest->nArg = (u16)nArg; diff --git a/lib/libsqlite3/src/delete.c b/lib/libsqlite3/src/delete.c index f666b90c835..eead4856b14 100644 --- a/lib/libsqlite3/src/delete.c +++ b/lib/libsqlite3/src/delete.c @@ -374,7 +374,7 @@ void sqlite3DeleteFrom( pParse, pTabList, pWhere, 0, 0, WHERE_DUPLICATES_OK ); if( pWInfo==0 ) goto delete_from_cleanup; - regRowid = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, iRowid); + regRowid = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, iRowid, 0); sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, regRowid); if( db->flags & SQLITE_CountRows ){ sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1); diff --git a/lib/libsqlite3/src/expr.c b/lib/libsqlite3/src/expr.c index 79dd8f49618..10d9f775d9c 100644 --- a/lib/libsqlite3/src/expr.c +++ b/lib/libsqlite3/src/expr.c @@ -484,8 +484,14 @@ Expr *sqlite3PExpr( Expr *pRight, /* Right operand */ const Token *pToken /* Argument token */ ){ - Expr *p = sqlite3ExprAlloc(pParse->db, op, pToken, 1); - sqlite3ExprAttachSubtrees(pParse->db, p, pLeft, pRight); + Expr *p; + if( op==TK_AND && pLeft && pRight ){ + /* Take advantage of short-circuit false optimization for AND */ + p = sqlite3ExprAnd(pParse->db, pLeft, pRight); + }else{ + p = sqlite3ExprAlloc(pParse->db, op, pToken, 1); + sqlite3ExprAttachSubtrees(pParse->db, p, pLeft, pRight); + } if( p ) { sqlite3ExprCheckHeight(pParse, p->nHeight); } @@ -493,14 +499,40 @@ Expr *sqlite3PExpr( } /* +** Return 1 if an expression must be FALSE in all cases and 0 if the +** expression might be true. This is an optimization. If is OK to +** return 0 here even if the expression really is always false (a +** false negative). But it is a bug to return 1 if the expression +** might be true in some rare circumstances (a false positive.) +** +** Note that if the expression is part of conditional for a +** LEFT JOIN, then we cannot determine at compile-time whether or not +** is it true or false, so always return 0. +*/ +static int exprAlwaysFalse(Expr *p){ + int v = 0; + if( ExprHasProperty(p, EP_FromJoin) ) return 0; + if( !sqlite3ExprIsInteger(p, &v) ) return 0; + return v==0; +} + +/* ** Join two expressions using an AND operator. If either expression is ** NULL, then just return the other expression. +** +** If one side or the other of the AND is known to be false, then instead +** of returning an AND expression, just return a constant expression with +** a value of false. */ Expr *sqlite3ExprAnd(sqlite3 *db, Expr *pLeft, Expr *pRight){ if( pLeft==0 ){ return pRight; }else if( pRight==0 ){ return pLeft; + }else if( exprAlwaysFalse(pLeft) || exprAlwaysFalse(pRight) ){ + sqlite3ExprDelete(db, pLeft); + sqlite3ExprDelete(db, pRight); + return sqlite3ExprAlloc(db, TK_INTEGER, &sqlite3IntTokens[0], 0); }else{ Expr *pNew = sqlite3ExprAlloc(db, TK_AND, 0, 0); sqlite3ExprAttachSubtrees(db, pNew, pLeft, pRight); @@ -2032,15 +2064,6 @@ void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int iReg){ */ #ifndef NDEBUG for(i=0, p=pParse->aColCache; i<SQLITE_N_COLCACHE; i++, p++){ -#if 0 /* This code wold remove the entry from the cache if it existed */ - if( p->iReg && p->iTable==iTab && p->iColumn==iCol ){ - cacheEntryClear(pParse, p); - p->iLevel = pParse->iCacheLevel; - p->iReg = iReg; - p->lru = pParse->iCacheCnt++; - return; - } -#endif assert( p->iReg==0 || p->iTable!=iTab || p->iColumn!=iCol ); } #endif @@ -2175,7 +2198,8 @@ int sqlite3ExprCodeGetColumn( Table *pTab, /* Description of the table we are reading from */ int iColumn, /* Index of the table column */ int iTable, /* The cursor pointing to the table */ - int iReg /* Store results here */ + int iReg, /* Store results here */ + u8 p5 /* P5 value for OP_Column */ ){ Vdbe *v = pParse->pVdbe; int i; @@ -2190,7 +2214,11 @@ int sqlite3ExprCodeGetColumn( } assert( v!=0 ); sqlite3ExprCodeGetColumnOfTable(v, pTab, iTable, iColumn, iReg); - sqlite3ExprCacheStore(pParse, iTable, iColumn, iReg); + if( p5 ){ + sqlite3VdbeChangeP5(v, p5); + }else{ + sqlite3ExprCacheStore(pParse, iTable, iColumn, iReg); + } return iReg; } @@ -2318,7 +2346,8 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ inReg = pExpr->iColumn + pParse->ckBase; }else{ inReg = sqlite3ExprCodeGetColumn(pParse, pExpr->pTab, - pExpr->iColumn, pExpr->iTable, target); + pExpr->iColumn, pExpr->iTable, target, + pExpr->op2); } break; } @@ -2595,6 +2624,25 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ if( pFarg ){ r1 = sqlite3GetTempRange(pParse, nFarg); + + /* For length() and typeof() functions with a column argument, + ** set the P5 parameter to the OP_Column opcode to OPFLAG_LENGTHARG + ** or OPFLAG_TYPEOFARG respectively, to avoid unnecessary data + ** loading. + */ + if( (pDef->flags & (SQLITE_FUNC_LENGTH|SQLITE_FUNC_TYPEOF))!=0 ){ + u8 exprOp; + assert( nFarg==1 ); + assert( pFarg->a[0].pExpr!=0 ); + exprOp = pFarg->a[0].pExpr->op; + if( exprOp==TK_COLUMN || exprOp==TK_AGG_COLUMN ){ + assert( SQLITE_FUNC_LENGTH==OPFLAG_LENGTHARG ); + assert( SQLITE_FUNC_TYPEOF==OPFLAG_TYPEOFARG ); + testcase( pDef->flags==SQLITE_FUNC_LENGTH ); + pFarg->a[0].pExpr->op2 = pDef->flags; + } + } + sqlite3ExprCachePush(pParse); /* Ticket 2ea2425d34be */ sqlite3ExprCodeExprList(pParse, pFarg, r1, 1); sqlite3ExprCachePop(pParse, 1); /* Ticket 2ea2425d34be */ @@ -3730,7 +3778,7 @@ int sqlite3ExprCompare(Expr *pA, Expr *pB){ if( !ExprHasProperty(pB, EP_IntValue) || pA->u.iValue!=pB->u.iValue ){ return 2; } - }else if( pA->op!=TK_COLUMN && pA->u.zToken ){ + }else if( pA->op!=TK_COLUMN && pA->op!=TK_AGG_COLUMN && pA->u.zToken ){ if( ExprHasProperty(pB, EP_IntValue) || NEVER(pB->u.zToken==0) ) return 2; if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){ return 2; @@ -3768,6 +3816,41 @@ int sqlite3ExprListCompare(ExprList *pA, ExprList *pB){ } /* +** This is the expression callback for sqlite3FunctionUsesOtherSrc(). +** +** Determine if an expression references any table other than one of the +** tables in pWalker->u.pSrcList and abort if it does. +*/ +static int exprUsesOtherSrc(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN ){ + int i; + SrcList *pSrc = pWalker->u.pSrcList; + for(i=0; i<pSrc->nSrc; i++){ + if( pExpr->iTable==pSrc->a[i].iCursor ) return WRC_Continue; + } + return WRC_Abort; + }else{ + return WRC_Continue; + } +} + +/* +** Determine if any of the arguments to the pExpr Function references +** any SrcList other than pSrcList. Return true if they do. Return +** false if pExpr has no argument or has only constant arguments or +** only references tables named in pSrcList. +*/ +static int sqlite3FunctionUsesOtherSrc(Expr *pExpr, SrcList *pSrcList){ + Walker w; + assert( pExpr->op==TK_AGG_FUNCTION ); + memset(&w, 0, sizeof(w)); + w.xExprCallback = exprUsesOtherSrc; + w.u.pSrcList = pSrcList; + if( sqlite3WalkExprList(&w, pExpr->x.pList)!=WRC_Continue ) return 1; + return 0; +} + +/* ** Add a new element to the pAggInfo->aCol[] array. Return the index of ** the new element. Return a negative number if malloc fails. */ @@ -3882,9 +3965,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ return WRC_Prune; } case TK_AGG_FUNCTION: { - /* The pNC->nDepth==0 test causes aggregate functions in subqueries - ** to be ignored */ - if( pNC->nDepth==0 ){ + if( !sqlite3FunctionUsesOtherSrc(pExpr, pSrcList) ){ /* Check to see if pExpr is a duplicate of another aggregate ** function that is already in the pAggInfo structure */ @@ -3928,15 +4009,9 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ return WRC_Continue; } static int analyzeAggregatesInSelect(Walker *pWalker, Select *pSelect){ - NameContext *pNC = pWalker->u.pNC; - if( pNC->nDepth==0 ){ - pNC->nDepth++; - sqlite3WalkSelect(pWalker, pSelect); - pNC->nDepth--; - return WRC_Prune; - }else{ - return WRC_Continue; - } + UNUSED_PARAMETER(pWalker); + UNUSED_PARAMETER(pSelect); + return WRC_Continue; } /* @@ -3949,6 +4024,7 @@ static int analyzeAggregatesInSelect(Walker *pWalker, Select *pSelect){ */ void sqlite3ExprAnalyzeAggregates(NameContext *pNC, Expr *pExpr){ Walker w; + memset(&w, 0, sizeof(w)); w.xExprCallback = analyzeAggregate; w.xSelectCallback = analyzeAggregatesInSelect; w.u.pNC = pNC; diff --git a/lib/libsqlite3/src/func.c b/lib/libsqlite3/src/func.c index c66ad28aba5..6ffc7184b04 100644 --- a/lib/libsqlite3/src/func.c +++ b/lib/libsqlite3/src/func.c @@ -1542,8 +1542,8 @@ void sqlite3RegisterGlobalFunctions(void){ FUNCTION(max, -1, 1, 1, minmaxFunc ), FUNCTION(max, 0, 1, 1, 0 ), AGGREGATE(max, 1, 1, 1, minmaxStep, minMaxFinalize ), - FUNCTION(typeof, 1, 0, 0, typeofFunc ), - FUNCTION(length, 1, 0, 0, lengthFunc ), + FUNCTION2(typeof, 1, 0, 0, typeofFunc, SQLITE_FUNC_TYPEOF), + FUNCTION2(length, 1, 0, 0, lengthFunc, SQLITE_FUNC_LENGTH), FUNCTION(substr, 2, 0, 0, substrFunc ), FUNCTION(substr, 3, 0, 0, substrFunc ), FUNCTION(abs, 1, 0, 0, absFunc ), @@ -1555,11 +1555,9 @@ void sqlite3RegisterGlobalFunctions(void){ FUNCTION(lower, 1, 0, 0, lowerFunc ), FUNCTION(coalesce, 1, 0, 0, 0 ), FUNCTION(coalesce, 0, 0, 0, 0 ), -/* FUNCTION(coalesce, -1, 0, 0, ifnullFunc ), */ - {-1,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"coalesce",0,0}, + FUNCTION2(coalesce, -1, 0, 0, ifnullFunc, SQLITE_FUNC_COALESCE), FUNCTION(hex, 1, 0, 0, hexFunc ), -/* FUNCTION(ifnull, 2, 0, 0, ifnullFunc ), */ - {2,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"ifnull",0,0}, + FUNCTION2(ifnull, 2, 0, 0, ifnullFunc, SQLITE_FUNC_COALESCE), FUNCTION(random, 0, 0, 0, randomFunc ), FUNCTION(randomblob, 1, 0, 0, randomBlob ), FUNCTION(nullif, 2, 0, 1, nullifFunc ), diff --git a/lib/libsqlite3/src/insert.c b/lib/libsqlite3/src/insert.c index 6b31e24f2c0..a589c8aef69 100644 --- a/lib/libsqlite3/src/insert.c +++ b/lib/libsqlite3/src/insert.c @@ -1157,9 +1157,11 @@ void sqlite3GenerateConstraintChecks( int regData; /* Register containing first data column */ int iCur; /* Table cursor number */ Index *pIdx; /* Pointer to one of the indices */ + sqlite3 *db; /* Database connection */ int seenReplace = 0; /* True if REPLACE is used to resolve INT PK conflict */ int regOldRowid = (rowidChng && isUpdate) ? rowidChng : regRowid; + db = pParse->db; v = sqlite3GetVdbe(pParse); assert( v!=0 ); assert( pTab->pSelect==0 ); /* This table is not a VIEW */ @@ -1192,7 +1194,7 @@ void sqlite3GenerateConstraintChecks( char *zMsg; sqlite3VdbeAddOp3(v, OP_HaltIfNull, SQLITE_CONSTRAINT, onError, regData+i); - zMsg = sqlite3MPrintf(pParse->db, "%s.%s may not be NULL", + zMsg = sqlite3MPrintf(db, "%s.%s may not be NULL", pTab->zName, pTab->aCol[i].zName); sqlite3VdbeChangeP4(v, -1, zMsg, P4_DYNAMIC); break; @@ -1214,18 +1216,27 @@ void sqlite3GenerateConstraintChecks( /* Test all CHECK constraints */ #ifndef SQLITE_OMIT_CHECK - if( pTab->pCheck && (pParse->db->flags & SQLITE_IgnoreChecks)==0 ){ - int allOk = sqlite3VdbeMakeLabel(v); + if( pTab->pCheck && (db->flags & SQLITE_IgnoreChecks)==0 ){ + ExprList *pCheck = pTab->pCheck; pParse->ckBase = regData; - sqlite3ExprIfTrue(pParse, pTab->pCheck, allOk, SQLITE_JUMPIFNULL); onError = overrideError!=OE_Default ? overrideError : OE_Abort; - if( onError==OE_Ignore ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); - }else{ - if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-15569-63625 */ - sqlite3HaltConstraint(pParse, onError, 0, 0); + for(i=0; i<pCheck->nExpr; i++){ + int allOk = sqlite3VdbeMakeLabel(v); + sqlite3ExprIfTrue(pParse, pCheck->a[i].pExpr, allOk, SQLITE_JUMPIFNULL); + if( onError==OE_Ignore ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); + }else{ + char *zConsName = pCheck->a[i].zName; + if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-15569-63625 */ + if( zConsName ){ + zConsName = sqlite3MPrintf(db, "constraint %s failed", zConsName); + }else{ + zConsName = 0; + } + sqlite3HaltConstraint(pParse, onError, zConsName, P4_DYNAMIC); + } + sqlite3VdbeResolveLabel(v, allOk); } - sqlite3VdbeResolveLabel(v, allOk); } #endif /* !defined(SQLITE_OMIT_CHECK) */ @@ -1281,7 +1292,7 @@ void sqlite3GenerateConstraintChecks( ** table. */ Trigger *pTrigger = 0; - if( pParse->db->flags&SQLITE_RecTriggers ){ + if( db->flags&SQLITE_RecTriggers ){ pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); } if( pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0) ){ @@ -1370,7 +1381,7 @@ void sqlite3GenerateConstraintChecks( char *zErr; sqlite3StrAccumInit(&errMsg, 0, 0, 200); - errMsg.db = pParse->db; + errMsg.db = db; zSep = pIdx->nColumn>1 ? "columns " : "column "; for(j=0; j<pIdx->nColumn; j++){ char *zCol = pTab->aCol[pIdx->aiColumn[j]].zName; @@ -1394,7 +1405,7 @@ void sqlite3GenerateConstraintChecks( Trigger *pTrigger = 0; assert( onError==OE_Replace ); sqlite3MultiWrite(pParse); - if( pParse->db->flags&SQLITE_RecTriggers ){ + if( db->flags&SQLITE_RecTriggers ){ pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); } sqlite3GenerateRowDelete( @@ -1724,7 +1735,7 @@ static int xferOptimization( } } #ifndef SQLITE_OMIT_CHECK - if( pDest->pCheck && sqlite3ExprCompare(pSrc->pCheck, pDest->pCheck) ){ + if( pDest->pCheck && sqlite3ExprListCompare(pSrc->pCheck, pDest->pCheck) ){ return 0; /* Tables have different CHECK constraints. Ticket #2252 */ } #endif diff --git a/lib/libsqlite3/src/os_unix.c b/lib/libsqlite3/src/os_unix.c index 48c130935e9..c85e9b53af4 100644 --- a/lib/libsqlite3/src/os_unix.c +++ b/lib/libsqlite3/src/os_unix.c @@ -165,8 +165,8 @@ #endif /* - ** Default permissions when creating auto proxy dir - */ +** Default permissions when creating auto proxy dir +*/ #ifndef SQLITE_DEFAULT_PROXYDIR_PERMISSIONS # define SQLITE_DEFAULT_PROXYDIR_PERMISSIONS 0755 #endif @@ -512,7 +512,7 @@ static const char *unixNextSystemCall(sqlite3_vfs *p, const char *zName){ /* ** Invoke open(). Do so multiple times, until it either succeeds or -** files for some reason other than EINTR. +** fails for some reason other than EINTR. ** ** If the file creation mode "m" is 0 then set it to the default for ** SQLite. The default is SQLITE_DEFAULT_FILE_PERMISSIONS (normally @@ -528,7 +528,7 @@ static const char *unixNextSystemCall(sqlite3_vfs *p, const char *zName){ ** recover the hot journals. */ static int robust_open(const char *z, int f, mode_t m){ - int rc; + int fd; mode_t m2; mode_t origM = 0; if( m==0 ){ @@ -537,11 +537,20 @@ static int robust_open(const char *z, int f, mode_t m){ m2 = m; origM = osUmask(0); } - do{ rc = osOpen(z,f,m2); }while( rc<0 && errno==EINTR ); + do{ +#if defined(O_CLOEXEC) + fd = osOpen(z,f|O_CLOEXEC,m2); +#else + fd = osOpen(z,f,m2); +#endif + }while( fd<0 && errno==EINTR ); if( m ){ osUmask(origM); } - return rc; +#if defined(FD_CLOEXEC) && (!defined(O_CLOEXEC) || O_CLOEXEC==0) + if( fd>=0 ) osFcntl(fd, F_SETFD, osFcntl(fd, F_GETFD, 0) | FD_CLOEXEC); +#endif + return fd; } /* @@ -3336,9 +3345,6 @@ static int openDirectory(const char *zFilename, int *pFd){ zDirname[ii] = '\0'; fd = robust_open(zDirname, O_RDONLY|O_BINARY, 0); if( fd>=0 ){ -#ifdef FD_CLOEXEC - osFcntl(fd, F_SETFD, osFcntl(fd, F_GETFD, 0) | FD_CLOEXEC); -#endif OSTRACE(("OPENDIR %-3d %s\n", fd, zDirname)); } } @@ -3421,7 +3427,7 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ ** actual file size after the operation may be larger than the requested ** size). */ - if( pFile->szChunk ){ + if( pFile->szChunk>0 ){ nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; } @@ -5183,10 +5189,6 @@ static int unixOpen( } #endif -#ifdef FD_CLOEXEC - osFcntl(fd, F_SETFD, osFcntl(fd, F_GETFD, 0) | FD_CLOEXEC); -#endif - noLock = eType!=SQLITE_OPEN_MAIN_DB; diff --git a/lib/libsqlite3/src/os_win.c b/lib/libsqlite3/src/os_win.c index 8b86c7ed59f..fcfe0118ed4 100644 --- a/lib/libsqlite3/src/os_win.c +++ b/lib/libsqlite3/src/os_win.c @@ -1616,6 +1616,9 @@ static int winClose(sqlite3_file *id){ } #endif OSTRACE(("CLOSE %d %s\n", pFile->h, rc ? "ok" : "failed")); + if( rc ){ + pFile->h = NULL; + } OpenCounter(-1); return rc ? SQLITE_OK : winLogError(SQLITE_IOERR_CLOSE, osGetLastError(), @@ -1633,6 +1636,9 @@ static int winRead( int amt, /* Number of bytes to read */ sqlite3_int64 offset /* Begin reading at this offset */ ){ +#if !SQLITE_OS_WINCE + OVERLAPPED overlapped; /* The offset for ReadFile. */ +#endif winFile *pFile = (winFile*)id; /* file handle */ DWORD nRead; /* Number of bytes actually read from file */ int nRetry = 0; /* Number of retrys */ @@ -1641,10 +1647,18 @@ static int winRead( SimulateIOError(return SQLITE_IOERR_READ); OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype)); +#if SQLITE_OS_WINCE if( seekWinFile(pFile, offset) ){ return SQLITE_FULL; } while( !osReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ +#else + memset(&overlapped, 0, sizeof(OVERLAPPED)); + overlapped.Offset = (LONG)(offset & 0xffffffff); + overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); + while( !osReadFile(pFile->h, pBuf, amt, &nRead, &overlapped) && + osGetLastError()!=ERROR_HANDLE_EOF ){ +#endif DWORD lastErrno; if( retryIoerr(&nRetry, &lastErrno) ) continue; pFile->lastErrno = lastErrno; @@ -1671,7 +1685,7 @@ static int winWrite( int amt, /* Number of bytes to write */ sqlite3_int64 offset /* Offset into the file to begin writing at */ ){ - int rc; /* True if error has occured, else false */ + int rc = 0; /* True if error has occured, else false */ winFile *pFile = (winFile*)id; /* File handle */ int nRetry = 0; /* Number of retries */ @@ -1682,19 +1696,44 @@ static int winWrite( OSTRACE(("WRITE %d lock=%d\n", pFile->h, pFile->locktype)); +#if SQLITE_OS_WINCE rc = seekWinFile(pFile, offset); if( rc==0 ){ +#else + { +#endif +#if !SQLITE_OS_WINCE + OVERLAPPED overlapped; /* The offset for WriteFile. */ +#endif u8 *aRem = (u8 *)pBuf; /* Data yet to be written */ int nRem = amt; /* Number of bytes yet to be written */ DWORD nWrite; /* Bytes written by each WriteFile() call */ DWORD lastErrno = NO_ERROR; /* Value returned by GetLastError() */ +#if !SQLITE_OS_WINCE + memset(&overlapped, 0, sizeof(OVERLAPPED)); + overlapped.Offset = (LONG)(offset & 0xffffffff); + overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); +#endif + while( nRem>0 ){ +#if SQLITE_OS_WINCE if( !osWriteFile(pFile->h, aRem, nRem, &nWrite, 0) ){ +#else + if( !osWriteFile(pFile->h, aRem, nRem, &nWrite, &overlapped) ){ +#endif if( retryIoerr(&nRetry, &lastErrno) ) continue; break; } - if( nWrite<=0 ) break; + if( nWrite<=0 ){ + lastErrno = osGetLastError(); + break; + } +#if !SQLITE_OS_WINCE + offset += nWrite; + overlapped.Offset = (LONG)(offset & 0xffffffff); + overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); +#endif aRem += nWrite; nRem -= nWrite; } @@ -2993,6 +3032,35 @@ static int getTempname(int nBuf, char *zBuf){ } /* +** Return TRUE if the named file is really a directory. Return false if +** it is something other than a directory, or if there is any kind of memory +** allocation failure. +*/ +static int winIsDir(const void *zConverted){ + DWORD attr; + int rc = 0; + DWORD lastErrno; + + if( isNT() ){ + int cnt = 0; + WIN32_FILE_ATTRIBUTE_DATA sAttrData; + memset(&sAttrData, 0, sizeof(sAttrData)); + while( !(rc = osGetFileAttributesExW((LPCWSTR)zConverted, + GetFileExInfoStandard, + &sAttrData)) && retryIoerr(&cnt, &lastErrno) ){} + if( !rc ){ + return 0; /* Invalid name? */ + } + attr = sAttrData.dwFileAttributes; +#if SQLITE_OS_WINCE==0 + }else{ + attr = osGetFileAttributesA((char*)zConverted); +#endif + } + return (attr!=INVALID_FILE_ATTRIBUTES) && (attr&FILE_ATTRIBUTE_DIRECTORY); +} + +/* ** Open a file. */ static int winOpen( @@ -3098,6 +3166,11 @@ static int winOpen( return SQLITE_IOERR_NOMEM; } + if( winIsDir(zConverted) ){ + sqlite3_free(zConverted); + return SQLITE_CANTOPEN_ISDIR; + } + if( isReadWrite ){ dwDesiredAccess = GENERIC_READ | GENERIC_WRITE; }else{ @@ -3147,11 +3220,9 @@ static int winOpen( dwCreationDisposition, dwFlagsAndAttributes, NULL))==INVALID_HANDLE_VALUE && - retryIoerr(&cnt, &lastErrno) ){} -/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. -** Since the ANSI version of these Windows API do not exist for WINCE, -** it's important to not reference them for WINCE builds. -*/ + retryIoerr(&cnt, &lastErrno) ){ + /* Noop */ + } #if SQLITE_OS_WINCE==0 }else{ while( (h = osCreateFileA((LPCSTR)zConverted, @@ -3160,7 +3231,9 @@ static int winOpen( dwCreationDisposition, dwFlagsAndAttributes, NULL))==INVALID_HANDLE_VALUE && - retryIoerr(&cnt, &lastErrno) ){} + retryIoerr(&cnt, &lastErrno) ){ + /* Noop */ + } #endif } @@ -3240,6 +3313,7 @@ static int winDelete( ){ int cnt = 0; int rc; + DWORD attr; DWORD lastErrno; void *zConverted; UNUSED_PARAMETER(pVfs); @@ -3251,20 +3325,50 @@ static int winDelete( return SQLITE_IOERR_NOMEM; } if( isNT() ){ - rc = 1; - while( osGetFileAttributesW(zConverted)!=INVALID_FILE_ATTRIBUTES && - (rc = osDeleteFileW(zConverted))==0 && retryIoerr(&cnt, &lastErrno) ){} - rc = rc ? SQLITE_OK : SQLITE_ERROR; + do { + attr = osGetFileAttributesW(zConverted); + if ( attr==INVALID_FILE_ATTRIBUTES ){ + rc = SQLITE_OK; /* Already gone? */ + break; + } + if ( attr&FILE_ATTRIBUTE_DIRECTORY ){ + rc = SQLITE_ERROR; /* Files only. */ + break; + } + if ( osDeleteFileW(zConverted) ){ + rc = SQLITE_OK; /* Deleted OK. */ + break; + } + if ( !retryIoerr(&cnt, &lastErrno) ){ + rc = SQLITE_ERROR; /* No more retries. */ + break; + } + } while(1); /* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. ** Since the ANSI version of these Windows API do not exist for WINCE, ** it's important to not reference them for WINCE builds. */ #if SQLITE_OS_WINCE==0 }else{ - rc = 1; - while( osGetFileAttributesA(zConverted)!=INVALID_FILE_ATTRIBUTES && - (rc = osDeleteFileA(zConverted))==0 && retryIoerr(&cnt, &lastErrno) ){} - rc = rc ? SQLITE_OK : SQLITE_ERROR; + do { + attr = osGetFileAttributesA(zConverted); + if ( attr==INVALID_FILE_ATTRIBUTES ){ + rc = SQLITE_OK; /* Already gone? */ + break; + } + if ( attr&FILE_ATTRIBUTE_DIRECTORY ){ + rc = SQLITE_ERROR; /* Files only. */ + break; + } + if ( osDeleteFileA(zConverted) ){ + rc = SQLITE_OK; /* Deleted OK. */ + break; + } + if ( !retryIoerr(&cnt, &lastErrno) ){ + rc = SQLITE_ERROR; /* No more retries. */ + break; + } + } while(1); #endif } if( rc ){ diff --git a/lib/libsqlite3/src/pager.c b/lib/libsqlite3/src/pager.c index 66252aa0f73..b93e0aa866c 100644 --- a/lib/libsqlite3/src/pager.c +++ b/lib/libsqlite3/src/pager.c @@ -670,9 +670,9 @@ struct Pager { char *zJournal; /* Name of the journal file */ int (*xBusyHandler)(void*); /* Function to call when busy */ void *pBusyHandlerArg; /* Context argument for xBusyHandler */ - int nHit, nMiss; /* Total cache hits and misses */ + int aStat[3]; /* Total cache hits, misses and writes */ #ifdef SQLITE_TEST - int nRead, nWrite; /* Database pages read/written */ + int nRead; /* Database pages read */ #endif void (*xReiniter)(DbPage*); /* Call this routine when reloading pages */ #ifdef SQLITE_HAS_CODEC @@ -690,6 +690,15 @@ struct Pager { }; /* +** Indexes for use with Pager.aStat[]. The Pager.aStat[] array contains +** the values accessed by passing SQLITE_DBSTATUS_CACHE_HIT, CACHE_MISS +** or CACHE_WRITE to sqlite3_db_status(). +*/ +#define PAGER_STAT_HIT 0 +#define PAGER_STAT_MISS 1 +#define PAGER_STAT_WRITE 2 + +/* ** The following global variables hold counters used for ** testing purposes only. These variables do not exist in ** a non-testing build. These variables are not thread-safe. @@ -2971,6 +2980,7 @@ static int pagerWalFrames( int isCommit /* True if this is a commit */ ){ int rc; /* Return code */ + int nList; /* Number of pages in pList */ #if defined(SQLITE_DEBUG) || defined(SQLITE_CHECK_PAGES) PgHdr *p; /* For looping over pages */ #endif @@ -2984,6 +2994,7 @@ static int pagerWalFrames( } #endif + assert( pList->pDirty==0 || isCommit ); if( isCommit ){ /* If a WAL transaction is being committed, there is no point in writing ** any pages with page numbers greater than nTruncate into the WAL file. @@ -2991,11 +3002,18 @@ static int pagerWalFrames( ** list here. */ PgHdr *p; PgHdr **ppNext = &pList; - for(p=pList; (*ppNext = p); p=p->pDirty){ - if( p->pgno<=nTruncate ) ppNext = &p->pDirty; + nList = 0; + for(p=pList; (*ppNext = p)!=0; p=p->pDirty){ + if( p->pgno<=nTruncate ){ + ppNext = &p->pDirty; + nList++; + } } assert( pList ); + }else{ + nList = 1; } + pPager->aStat[PAGER_STAT_WRITE] += nList; if( pList->pgno==1 ) pager_write_changecounter(pList); rc = sqlite3WalFrames(pPager->pWal, @@ -4063,6 +4081,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ if( pgno>pPager->dbFileSize ){ pPager->dbFileSize = pgno; } + pPager->aStat[PAGER_STAT_WRITE]++; /* Update any backup objects copying the contents of this pager. */ sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)pList->pData); @@ -4071,7 +4090,6 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ PAGERID(pPager), pgno, pager_pagehash(pList))); IOTRACE(("PGOUT %p %d\n", pPager, pgno)); PAGER_INCR(sqlite3_pager_writedb_count); - PAGER_INCR(pPager->nWrite); }else{ PAGERTRACE(("NOSTORE %d page %d\n", PAGERID(pPager), pgno)); } @@ -5029,7 +5047,7 @@ int sqlite3PagerAcquire( /* In this case the pcache already contains an initialized copy of ** the page. Return without further ado. */ assert( pgno<=PAGER_MAX_PGNO && pgno!=PAGER_MJ_PGNO(pPager) ); - pPager->nHit++; + pPager->aStat[PAGER_STAT_HIT]++; return SQLITE_OK; }else{ @@ -5071,7 +5089,7 @@ int sqlite3PagerAcquire( IOTRACE(("ZERO %p %d\n", pPager, pgno)); }else{ assert( pPg->pPager==pPager ); - pPager->nMiss++; + pPager->aStat[PAGER_STAT_MISS]++; rc = readDbPage(pPg); if( rc!=SQLITE_OK ){ goto pager_acquire_err; @@ -5656,6 +5674,7 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ CODEC2(pPager, pPgHdr->pData, 1, 6, rc=SQLITE_NOMEM, zBuf); if( rc==SQLITE_OK ){ rc = sqlite3OsWrite(pPager->fd, zBuf, pPager->pageSize, 0); + pPager->aStat[PAGER_STAT_WRITE]++; } if( rc==SQLITE_OK ){ pPager->changeCountDone = 1; @@ -6099,11 +6118,11 @@ int *sqlite3PagerStats(Pager *pPager){ a[3] = pPager->eState==PAGER_OPEN ? -1 : (int) pPager->dbSize; a[4] = pPager->eState; a[5] = pPager->errCode; - a[6] = pPager->nHit; - a[7] = pPager->nMiss; + a[6] = pPager->aStat[PAGER_STAT_HIT]; + a[7] = pPager->aStat[PAGER_STAT_MISS]; a[8] = 0; /* Used to be pPager->nOvfl */ a[9] = pPager->nRead; - a[10] = pPager->nWrite; + a[10] = pPager->aStat[PAGER_STAT_WRITE]; return a; } #endif @@ -6116,20 +6135,19 @@ int *sqlite3PagerStats(Pager *pPager){ ** returning. */ void sqlite3PagerCacheStat(Pager *pPager, int eStat, int reset, int *pnVal){ - int *piStat; assert( eStat==SQLITE_DBSTATUS_CACHE_HIT || eStat==SQLITE_DBSTATUS_CACHE_MISS + || eStat==SQLITE_DBSTATUS_CACHE_WRITE ); - if( eStat==SQLITE_DBSTATUS_CACHE_HIT ){ - piStat = &pPager->nHit; - }else{ - piStat = &pPager->nMiss; - } - *pnVal += *piStat; + assert( SQLITE_DBSTATUS_CACHE_HIT+1==SQLITE_DBSTATUS_CACHE_MISS ); + assert( SQLITE_DBSTATUS_CACHE_HIT+2==SQLITE_DBSTATUS_CACHE_WRITE ); + assert( PAGER_STAT_HIT==0 && PAGER_STAT_MISS==1 && PAGER_STAT_WRITE==2 ); + + *pnVal += pPager->aStat[eStat - SQLITE_DBSTATUS_CACHE_HIT]; if( reset ){ - *piStat = 0; + pPager->aStat[eStat - SQLITE_DBSTATUS_CACHE_HIT] = 0; } } diff --git a/lib/libsqlite3/src/resolve.c b/lib/libsqlite3/src/resolve.c index 3da48136fdf..090fa79842f 100644 --- a/lib/libsqlite3/src/resolve.c +++ b/lib/libsqlite3/src/resolve.c @@ -533,7 +533,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ nId = sqlite3Strlen30(zId); pDef = sqlite3FindFunction(pParse->db, zId, nId, n, enc, 0); if( pDef==0 ){ - pDef = sqlite3FindFunction(pParse->db, zId, nId, -1, enc, 0); + pDef = sqlite3FindFunction(pParse->db, zId, nId, -2, enc, 0); if( pDef==0 ){ no_such_func = 1; }else{ @@ -883,7 +883,7 @@ static int resolveOrderGroupBy( ExprList *pOrderBy, /* An ORDER BY or GROUP BY clause to resolve */ const char *zType /* Either "ORDER" or "GROUP", as appropriate */ ){ - int i; /* Loop counter */ + int i, j; /* Loop counters */ int iCol; /* Column number */ struct ExprList_item *pItem; /* A term of the ORDER BY clause */ Parse *pParse; /* Parsing context */ @@ -920,6 +920,11 @@ static int resolveOrderGroupBy( if( sqlite3ResolveExprNames(pNC, pE) ){ return 1; } + for(j=0; j<pSelect->pEList->nExpr; j++){ + if( sqlite3ExprCompare(pE, pSelect->pEList->a[j].pExpr)==0 ){ + pItem->iOrderByCol = j+1; + } + } } return sqlite3ResolveOrderGroupBy(pParse, pSelect, pOrderBy, zType); } diff --git a/lib/libsqlite3/src/rowset.c b/lib/libsqlite3/src/rowset.c index d84bb93abf0..58c18b78db1 100644 --- a/lib/libsqlite3/src/rowset.c +++ b/lib/libsqlite3/src/rowset.c @@ -76,6 +76,11 @@ /* ** Each entry in a RowSet is an instance of the following object. +** +** This same object is reused to store a linked list of trees of RowSetEntry +** objects. In that alternative use, pRight points to the next entry +** in the list, pLeft points to the tree, and v is unused. The +** RowSet.pForest value points to the head of this forest list. */ struct RowSetEntry { i64 v; /* ROWID value for this entry */ @@ -105,13 +110,19 @@ struct RowSet { struct RowSetEntry *pEntry; /* List of entries using pRight */ struct RowSetEntry *pLast; /* Last entry on the pEntry list */ struct RowSetEntry *pFresh; /* Source of new entry objects */ - struct RowSetEntry *pTree; /* Binary tree of entries */ + struct RowSetEntry *pForest; /* List of binary trees of entries */ u16 nFresh; /* Number of objects on pFresh */ - u8 isSorted; /* True if pEntry is sorted */ + u8 rsFlags; /* Various flags */ u8 iBatch; /* Current insert batch */ }; /* +** Allowed values for RowSet.rsFlags +*/ +#define ROWSET_SORTED 0x01 /* True if RowSet.pEntry is sorted */ +#define ROWSET_NEXT 0x02 /* True if sqlite3RowSetNext() has been called */ + +/* ** Turn bulk memory into a RowSet object. N bytes of memory ** are available at pSpace. The db pointer is used as a memory context ** for any subsequent allocations that need to occur. @@ -131,10 +142,10 @@ RowSet *sqlite3RowSetInit(sqlite3 *db, void *pSpace, unsigned int N){ p->db = db; p->pEntry = 0; p->pLast = 0; - p->pTree = 0; + p->pForest = 0; p->pFresh = (struct RowSetEntry*)(ROUND8(sizeof(*p)) + (char*)p); p->nFresh = (u16)((N - ROUND8(sizeof(*p)))/sizeof(struct RowSetEntry)); - p->isSorted = 1; + p->rsFlags = ROWSET_SORTED; p->iBatch = 0; return p; } @@ -154,43 +165,59 @@ void sqlite3RowSetClear(RowSet *p){ p->nFresh = 0; p->pEntry = 0; p->pLast = 0; - p->pTree = 0; - p->isSorted = 1; + p->pForest = 0; + p->rsFlags = ROWSET_SORTED; } /* -** Insert a new value into a RowSet. +** Allocate a new RowSetEntry object that is associated with the +** given RowSet. Return a pointer to the new and completely uninitialized +** objected. ** -** The mallocFailed flag of the database connection is set if a -** memory allocation fails. +** In an OOM situation, the RowSet.db->mallocFailed flag is set and this +** routine returns NULL. */ -void sqlite3RowSetInsert(RowSet *p, i64 rowid){ - struct RowSetEntry *pEntry; /* The new entry */ - struct RowSetEntry *pLast; /* The last prior entry */ +static struct RowSetEntry *rowSetEntryAlloc(RowSet *p){ assert( p!=0 ); if( p->nFresh==0 ){ struct RowSetChunk *pNew; pNew = sqlite3DbMallocRaw(p->db, sizeof(*pNew)); if( pNew==0 ){ - return; + return 0; } pNew->pNextChunk = p->pChunk; p->pChunk = pNew; p->pFresh = pNew->aEntry; p->nFresh = ROWSET_ENTRY_PER_CHUNK; } - pEntry = p->pFresh++; p->nFresh--; + return p->pFresh++; +} + +/* +** Insert a new value into a RowSet. +** +** The mallocFailed flag of the database connection is set if a +** memory allocation fails. +*/ +void sqlite3RowSetInsert(RowSet *p, i64 rowid){ + struct RowSetEntry *pEntry; /* The new entry */ + struct RowSetEntry *pLast; /* The last prior entry */ + + /* This routine is never called after sqlite3RowSetNext() */ + assert( p!=0 && (p->rsFlags & ROWSET_NEXT)==0 ); + + pEntry = rowSetEntryAlloc(p); + if( pEntry==0 ) return; pEntry->v = rowid; pEntry->pRight = 0; pLast = p->pLast; if( pLast ){ - if( p->isSorted && rowid<=pLast->v ){ - p->isSorted = 0; + if( (p->rsFlags & ROWSET_SORTED)!=0 && rowid<=pLast->v ){ + p->rsFlags &= ~ROWSET_SORTED; } pLast->pRight = pEntry; }else{ - assert( p->pEntry==0 ); /* Fires if INSERT after SMALLEST */ p->pEntry = pEntry; } p->pLast = pEntry; @@ -202,7 +229,7 @@ void sqlite3RowSetInsert(RowSet *p, i64 rowid){ ** The input lists are connected via pRight pointers and are ** assumed to each already be in sorted order. */ -static struct RowSetEntry *rowSetMerge( +static struct RowSetEntry *rowSetEntryMerge( struct RowSetEntry *pA, /* First sorted list to be merged */ struct RowSetEntry *pB /* Second sorted list to be merged */ ){ @@ -236,32 +263,29 @@ static struct RowSetEntry *rowSetMerge( } /* -** Sort all elements on the pEntry list of the RowSet into ascending order. +** Sort all elements on the list of RowSetEntry objects into order of +** increasing v. */ -static void rowSetSort(RowSet *p){ +static struct RowSetEntry *rowSetEntrySort(struct RowSetEntry *pIn){ unsigned int i; - struct RowSetEntry *pEntry; - struct RowSetEntry *aBucket[40]; + struct RowSetEntry *pNext, *aBucket[40]; - assert( p->isSorted==0 ); memset(aBucket, 0, sizeof(aBucket)); - while( p->pEntry ){ - pEntry = p->pEntry; - p->pEntry = pEntry->pRight; - pEntry->pRight = 0; + while( pIn ){ + pNext = pIn->pRight; + pIn->pRight = 0; for(i=0; aBucket[i]; i++){ - pEntry = rowSetMerge(aBucket[i], pEntry); + pIn = rowSetEntryMerge(aBucket[i], pIn); aBucket[i] = 0; } - aBucket[i] = pEntry; + aBucket[i] = pIn; + pIn = pNext; } - pEntry = 0; + pIn = 0; for(i=0; i<sizeof(aBucket)/sizeof(aBucket[0]); i++){ - pEntry = rowSetMerge(pEntry, aBucket[i]); + pIn = rowSetEntryMerge(pIn, aBucket[i]); } - p->pEntry = pEntry; - p->pLast = 0; - p->isSorted = 1; + return pIn; } @@ -355,20 +379,37 @@ static struct RowSetEntry *rowSetListToTree(struct RowSetEntry *pList){ } /* -** Convert the list in p->pEntry into a sorted list if it is not -** sorted already. If there is a binary tree on p->pTree, then -** convert it into a list too and merge it into the p->pEntry list. +** Take all the entries on p->pEntry and on the trees in p->pForest and +** sort them all together into one big ordered list on p->pEntry. +** +** This routine should only be called once in the life of a RowSet. */ static void rowSetToList(RowSet *p){ - if( !p->isSorted ){ - rowSetSort(p); + + /* This routine is called only once */ + assert( p!=0 && (p->rsFlags & ROWSET_NEXT)==0 ); + + if( (p->rsFlags & ROWSET_SORTED)==0 ){ + p->pEntry = rowSetEntrySort(p->pEntry); } - if( p->pTree ){ - struct RowSetEntry *pHead, *pTail; - rowSetTreeToList(p->pTree, &pHead, &pTail); - p->pTree = 0; - p->pEntry = rowSetMerge(p->pEntry, pHead); + + /* While this module could theoretically support it, sqlite3RowSetNext() + ** is never called after sqlite3RowSetText() for the same RowSet. So + ** there is never a forest to deal with. Should this change, simply + ** remove the assert() and the #if 0. */ + assert( p->pForest==0 ); +#if 0 + while( p->pForest ){ + struct RowSetEntry *pTree = p->pForest->pLeft; + if( pTree ){ + struct RowSetEntry *pHead, *pTail; + rowSetTreeToList(pTree, &pHead, &pTail); + p->pEntry = rowSetEntryMerge(p->pEntry, pHead); + } + p->pForest = p->pForest->pRight; } +#endif + p->rsFlags |= ROWSET_NEXT; /* Verify this routine is never called again */ } /* @@ -380,7 +421,12 @@ static void rowSetToList(RowSet *p){ ** routine may not be called again. */ int sqlite3RowSetNext(RowSet *p, i64 *pRowid){ - rowSetToList(p); + assert( p!=0 ); + + /* Merge the forest into a single sorted list on first call */ + if( (p->rsFlags & ROWSET_NEXT)==0 ) rowSetToList(p); + + /* Return the next entry on the list */ if( p->pEntry ){ *pRowid = p->pEntry->v; p->pEntry = p->pEntry->pRight; @@ -396,26 +442,66 @@ int sqlite3RowSetNext(RowSet *p, i64 *pRowid){ /* ** Check to see if element iRowid was inserted into the the rowset as ** part of any insert batch prior to iBatch. Return 1 or 0. +** +** If this is the first test of a new batch and if there exist entires +** on pRowSet->pEntry, then sort those entires into the forest at +** pRowSet->pForest so that they can be tested. */ int sqlite3RowSetTest(RowSet *pRowSet, u8 iBatch, sqlite3_int64 iRowid){ - struct RowSetEntry *p; + struct RowSetEntry *p, *pTree; + + /* This routine is never called after sqlite3RowSetNext() */ + assert( pRowSet!=0 && (pRowSet->rsFlags & ROWSET_NEXT)==0 ); + + /* Sort entries into the forest on the first test of a new batch + */ if( iBatch!=pRowSet->iBatch ){ - if( pRowSet->pEntry ){ - rowSetToList(pRowSet); - pRowSet->pTree = rowSetListToTree(pRowSet->pEntry); + p = pRowSet->pEntry; + if( p ){ + struct RowSetEntry **ppPrevTree = &pRowSet->pForest; + if( (pRowSet->rsFlags & ROWSET_SORTED)==0 ){ + p = rowSetEntrySort(p); + } + for(pTree = pRowSet->pForest; pTree; pTree=pTree->pRight){ + ppPrevTree = &pTree->pRight; + if( pTree->pLeft==0 ){ + pTree->pLeft = rowSetListToTree(p); + break; + }else{ + struct RowSetEntry *pAux, *pTail; + rowSetTreeToList(pTree->pLeft, &pAux, &pTail); + pTree->pLeft = 0; + p = rowSetEntryMerge(pAux, p); + } + } + if( pTree==0 ){ + *ppPrevTree = pTree = rowSetEntryAlloc(pRowSet); + if( pTree ){ + pTree->v = 0; + pTree->pRight = 0; + pTree->pLeft = rowSetListToTree(p); + } + } pRowSet->pEntry = 0; pRowSet->pLast = 0; + pRowSet->rsFlags |= ROWSET_SORTED; } pRowSet->iBatch = iBatch; } - p = pRowSet->pTree; - while( p ){ - if( p->v<iRowid ){ - p = p->pRight; - }else if( p->v>iRowid ){ - p = p->pLeft; - }else{ - return 1; + + /* Test to see if the iRowid value appears anywhere in the forest. + ** Return 1 if it does and 0 if not. + */ + for(pTree = pRowSet->pForest; pTree; pTree=pTree->pRight){ + p = pTree->pLeft; + while( p ){ + if( p->v<iRowid ){ + p = p->pRight; + }else if( p->v>iRowid ){ + p = p->pLeft; + }else{ + return 1; + } } } return 0; diff --git a/lib/libsqlite3/src/select.c b/lib/libsqlite3/src/select.c index 3efe014d8b6..835d9fd9ccb 100644 --- a/lib/libsqlite3/src/select.c +++ b/lib/libsqlite3/src/select.c @@ -1258,9 +1258,17 @@ static int selectColumnsFromExprList( char *zName; /* Column name */ int nName; /* Size of name in zName[] */ - *pnCol = nCol = pEList ? pEList->nExpr : 0; - aCol = *paCol = sqlite3DbMallocZero(db, sizeof(aCol[0])*nCol); - if( aCol==0 ) return SQLITE_NOMEM; + if( pEList ){ + nCol = pEList->nExpr; + aCol = sqlite3DbMallocZero(db, sizeof(aCol[0])*nCol); + testcase( aCol==0 ); + }else{ + nCol = 0; + aCol = 0; + } + *pnCol = nCol; + *paCol = aCol; + for(i=0, pCol=aCol; i<nCol; i++, pCol++){ /* Get an appropriate name for the column */ @@ -2843,7 +2851,8 @@ static int flattenSubquery( /* Authorize the subquery */ pParse->zAuthContext = pSubitem->zName; - sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0); + TESTONLY(i =) sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0); + testcase( i==SQLITE_DENY ); pParse->zAuthContext = zSavedAuthContext; /* If the sub-query is a compound SELECT statement, then (by restrictions @@ -4228,7 +4237,7 @@ int sqlite3Select( int r2; r2 = sqlite3ExprCodeGetColumn(pParse, - pCol->pTab, pCol->iColumn, pCol->iTable, r1); + pCol->pTab, pCol->iColumn, pCol->iTable, r1, 0); if( r1!=r2 ){ sqlite3VdbeAddOp2(v, OP_SCopy, r2, r1); } diff --git a/lib/libsqlite3/src/sqlite.h.in b/lib/libsqlite3/src/sqlite.h.in index 15a1b603a35..29355d77075 100644 --- a/lib/libsqlite3/src/sqlite.h.in +++ b/lib/libsqlite3/src/sqlite.h.in @@ -453,6 +453,7 @@ int sqlite3_exec( #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) +#define SQLITE_CANTOPEN_ISDIR (SQLITE_CANTOPEN | (2<<8)) #define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) #define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) #define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) @@ -766,7 +767,7 @@ struct sqlite3_io_methods { ** ** <li>[[SQLITE_FCNTL_PERSIST_WAL]] ** ^The [SQLITE_FCNTL_PERSIST_WAL] opcode is used to set or query the -** persistent [WAL | Write AHead Log] setting. By default, the auxiliary +** persistent [WAL | Write Ahead Log] setting. By default, the auxiliary ** write ahead log and shared memory files used for transaction control ** are automatically deleted when the latest connection to the database ** closes. Setting persistent WAL mode causes those files to persist after @@ -1542,7 +1543,7 @@ struct sqlite3_mem_methods { ** [SQLITE_USE_URI] symbol defined. ** ** [[SQLITE_CONFIG_PCACHE]] [[SQLITE_CONFIG_GETPCACHE]] -** <dt>SQLITE_CONFIG_PCACHE and SQLITE_CONFNIG_GETPCACHE +** <dt>SQLITE_CONFIG_PCACHE and SQLITE_CONFIG_GETPCACHE ** <dd> These options are obsolete and should not be used by new code. ** They are retained for backwards compatibility but are now no-ops. ** </dl> @@ -6001,6 +6002,17 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); ** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_MISS ** is always 0. ** </dd> +** +** [[SQLITE_DBSTATUS_CACHE_WRITE]] ^(<dt>SQLITE_DBSTATUS_CACHE_WRITE</dt> +** <dd>This parameter returns the number of dirty cache entries that have +** been written to disk. Specifically, the number of pages written to the +** wal file in wal mode databases, or the number of pages written to the +** database file in rollback mode databases. Any pages written as part of +** transaction rollback or database recovery operations are not included. +** If an IO or other error occurs while writing a page to disk, the effect +** on subsequent SQLITE_DBSTATUS_CACHE_WRITE requests is undefined.)^ ^The +** highwater mark associated with SQLITE_DBSTATUS_CACHE_WRITE is always 0. +** </dd> ** </dl> */ #define SQLITE_DBSTATUS_LOOKASIDE_USED 0 @@ -6012,7 +6024,8 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); #define SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL 6 #define SQLITE_DBSTATUS_CACHE_HIT 7 #define SQLITE_DBSTATUS_CACHE_MISS 8 -#define SQLITE_DBSTATUS_MAX 8 /* Largest defined DBSTATUS */ +#define SQLITE_DBSTATUS_CACHE_WRITE 9 +#define SQLITE_DBSTATUS_MAX 9 /* Largest defined DBSTATUS */ /* diff --git a/lib/libsqlite3/src/sqliteInt.h b/lib/libsqlite3/src/sqliteInt.h index 15c3f302e2a..cb178ff97cc 100644 --- a/lib/libsqlite3/src/sqliteInt.h +++ b/lib/libsqlite3/src/sqliteInt.h @@ -1009,14 +1009,18 @@ struct FuncDestructor { }; /* -** Possible values for FuncDef.flags +** Possible values for FuncDef.flags. Note that the _LENGTH and _TYPEOF +** values must correspond to OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG. There +** are assert() statements in the code to verify this. */ #define SQLITE_FUNC_LIKE 0x01 /* Candidate for the LIKE optimization */ #define SQLITE_FUNC_CASE 0x02 /* Case-sensitive LIKE-type function */ #define SQLITE_FUNC_EPHEM 0x04 /* Ephemeral. Delete with VDBE */ #define SQLITE_FUNC_NEEDCOLL 0x08 /* sqlite3GetFuncCollSeq() might be called */ -#define SQLITE_FUNC_COUNT 0x20 /* Built-in count(*) aggregate */ -#define SQLITE_FUNC_COALESCE 0x40 /* Built-in coalesce() or ifnull() function */ +#define SQLITE_FUNC_COUNT 0x10 /* Built-in count(*) aggregate */ +#define SQLITE_FUNC_COALESCE 0x20 /* Built-in coalesce() or ifnull() function */ +#define SQLITE_FUNC_LENGTH 0x40 /* Built-in length() function */ +#define SQLITE_FUNC_TYPEOF 0x80 /* Built-in typeof() function */ /* ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are @@ -1044,7 +1048,10 @@ struct FuncDestructor { ** parameter. */ #define FUNCTION(zName, nArg, iArg, bNC, xFunc) \ - {nArg, SQLITE_UTF8, bNC*SQLITE_FUNC_NEEDCOLL, \ + {nArg, SQLITE_UTF8, (bNC*SQLITE_FUNC_NEEDCOLL), \ + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} +#define FUNCTION2(zName, nArg, iArg, bNC, xFunc, extraFlags) \ + {nArg, SQLITE_UTF8, (bNC*SQLITE_FUNC_NEEDCOLL)|extraFlags, \ SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \ {nArg, SQLITE_UTF8, bNC*SQLITE_FUNC_NEEDCOLL, \ @@ -1274,7 +1281,7 @@ struct Table { FKey *pFKey; /* Linked list of all foreign keys in this table */ char *zColAff; /* String defining the affinity of each column */ #ifndef SQLITE_OMIT_CHECK - Expr *pCheck; /* The AND of all CHECK constraints */ + ExprList *pCheck; /* All CHECK constraints */ #endif #ifndef SQLITE_OMIT_ALTERTABLE int addColOffset; /* Offset in CREATE TABLE stmt to add a new column */ @@ -1667,6 +1674,7 @@ struct Expr { i16 iRightJoinTable; /* If EP_FromJoin, the right table of the join */ u8 flags2; /* Second set of flags. EP2_... */ u8 op2; /* If a TK_REGISTER, the original value of Expr.op */ + /* If TK_COLUMN, the value of p5 for OP_Column */ AggInfo *pAggInfo; /* Used by TK_AGG_COLUMN and TK_AGG_FUNCTION */ Table *pTab; /* Table for TK_COLUMN expressions. */ #if SQLITE_MAX_EXPR_DEPTH>0 @@ -1689,7 +1697,7 @@ struct Expr { #define EP_FixedDest 0x0200 /* Result needed in a specific register */ #define EP_IntValue 0x0400 /* Integer value contained in u.iValue */ #define EP_xIsSelect 0x0800 /* x.pSelect is valid (otherwise x.pList is) */ -#define EP_Hint 0x1000 /* Optimizer hint. Not required for correctness */ +#define EP_Hint 0x1000 /* Not used */ #define EP_Reduced 0x2000 /* Expr struct is EXPR_REDUCEDSIZE bytes only */ #define EP_TokenOnly 0x4000 /* Expr struct is EXPR_TOKENONLYSIZE bytes only */ #define EP_Static 0x8000 /* Held in memory not obtained from malloc() */ @@ -2003,7 +2011,6 @@ struct NameContext { u8 allowAgg; /* Aggregate functions allowed here */ u8 hasAgg; /* True if aggregates are seen */ u8 isCheck; /* True if resolving names in a CHECK constraint */ - int nDepth; /* Depth of subquery recursion. 1 for no recursion */ AggInfo *pAggInfo; /* Information about aggregates at this level */ NameContext *pNext; /* Next outer name context. NULL for outermost */ }; @@ -2208,6 +2215,7 @@ struct Parse { int regRowid; /* Register holding rowid of CREATE TABLE entry */ int regRoot; /* Register holding root page number for new objects */ int nMaxArg; /* Max args passed to user function by sub-program */ + Token constraintName;/* Name of the constraint currently being parsed */ #ifndef SQLITE_OMIT_SHARED_CACHE int nTableLock; /* Number of locks in aTableLock */ TableLock *aTableLock; /* Required table locks for shared-cache mode */ @@ -2276,7 +2284,7 @@ struct AuthContext { }; /* -** Bitfield flags for P5 value in OP_Insert and OP_Delete +** Bitfield flags for P5 value in various opcodes. */ #define OPFLAG_NCHANGE 0x01 /* Set to update db->nChange */ #define OPFLAG_LASTROWID 0x02 /* Set to update db->lastRowid */ @@ -2284,6 +2292,8 @@ struct AuthContext { #define OPFLAG_APPEND 0x08 /* This is likely to be an append */ #define OPFLAG_USESEEKRESULT 0x10 /* Try to avoid a seek in BtreeInsert() */ #define OPFLAG_CLEARCACHE 0x20 /* Clear pseudo-table cache in OP_Column */ +#define OPFLAG_LENGTHARG 0x40 /* OP_Column only used for length() */ +#define OPFLAG_TYPEOFARG 0x80 /* OP_Column only used for typeof() */ /* * Each trigger present in the database schema is stored as an instance of @@ -2466,6 +2476,7 @@ struct Walker { union { /* Extra data for callback */ NameContext *pNC; /* Naming context */ int i; /* Integer value */ + SrcList *pSrcList; /* FROM clause */ } u; }; @@ -2767,7 +2778,7 @@ void sqlite3DeleteFrom(Parse*, SrcList*, Expr*); void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int); WhereInfo *sqlite3WhereBegin(Parse*, SrcList*, Expr*, ExprList**,ExprList*,u16); void sqlite3WhereEnd(WhereInfo*); -int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int); +int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, u8); void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); void sqlite3ExprCodeMove(Parse*, int, int, int); void sqlite3ExprCodeCopy(Parse*, int, int, int); @@ -2834,7 +2845,7 @@ SrcList *sqlite3SrcListDup(sqlite3*,SrcList*,int); IdList *sqlite3IdListDup(sqlite3*,IdList*); Select *sqlite3SelectDup(sqlite3*,Select*,int); void sqlite3FuncDefInsert(FuncDefHash*, FuncDef*); -FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,int,u8,int); +FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,int,u8,u8); void sqlite3RegisterBuiltinFunctions(sqlite3*); void sqlite3RegisterDateTimeFunctions(void); void sqlite3RegisterGlobalFunctions(void); diff --git a/lib/libsqlite3/src/status.c b/lib/libsqlite3/src/status.c index b23238bb145..04b7656afe8 100644 --- a/lib/libsqlite3/src/status.c +++ b/lib/libsqlite3/src/status.c @@ -224,10 +224,12 @@ int sqlite3_db_status( ** to zero. */ case SQLITE_DBSTATUS_CACHE_HIT: - case SQLITE_DBSTATUS_CACHE_MISS: { + case SQLITE_DBSTATUS_CACHE_MISS: + case SQLITE_DBSTATUS_CACHE_WRITE:{ int i; int nRet = 0; assert( SQLITE_DBSTATUS_CACHE_MISS==SQLITE_DBSTATUS_CACHE_HIT+1 ); + assert( SQLITE_DBSTATUS_CACHE_WRITE==SQLITE_DBSTATUS_CACHE_HIT+2 ); for(i=0; i<db->nDb; i++){ if( db->aDb[i].pBt ){ diff --git a/lib/libsqlite3/src/tclsqlite.c b/lib/libsqlite3/src/tclsqlite.c index 967b1a004f8..51f8c517dfb 100644 --- a/lib/libsqlite3/src/tclsqlite.c +++ b/lib/libsqlite3/src/tclsqlite.c @@ -1163,7 +1163,7 @@ static int dbPrepareAndBind( memset(pPreStmt, 0, nByte); pPreStmt->pStmt = pStmt; - pPreStmt->nSql = (*pzOut - zSql); + pPreStmt->nSql = (int)(*pzOut - zSql); pPreStmt->zSql = sqlite3_sql(pStmt); pPreStmt->apParm = (Tcl_Obj **)&pPreStmt[1]; #ifdef SQLITE_TEST @@ -3108,23 +3108,19 @@ EXTERN int Sqlite3_Init(Tcl_Interp *interp){ return TCL_OK; } EXTERN int Tclsqlite3_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp); } -EXTERN int Sqlite3_SafeInit(Tcl_Interp *interp){ return TCL_OK; } -EXTERN int Tclsqlite3_SafeInit(Tcl_Interp *interp){ return TCL_OK; } EXTERN int Sqlite3_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } EXTERN int Tclsqlite3_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } -EXTERN int Sqlite3_SafeUnload(Tcl_Interp *interp, int flags){ return TCL_OK; } -EXTERN int Tclsqlite3_SafeUnload(Tcl_Interp *interp, int flags){ return TCL_OK;} +/* Because it accesses the file-system and uses persistent state, SQLite +** is not considered appropriate for safe interpreters. Hence, we deliberately +** omit the _SafeInit() interfaces. +*/ #ifndef SQLITE_3_SUFFIX_ONLY int Sqlite_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp); } int Tclsqlite_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp); } -int Sqlite_SafeInit(Tcl_Interp *interp){ return TCL_OK; } -int Tclsqlite_SafeInit(Tcl_Interp *interp){ return TCL_OK; } int Sqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } int Tclsqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } -int Sqlite_SafeUnload(Tcl_Interp *interp, int flags){ return TCL_OK; } -int Tclsqlite_SafeUnload(Tcl_Interp *interp, int flags){ return TCL_OK;} #endif #ifdef TCLSH @@ -3474,7 +3470,7 @@ static int md5file_cmd(void*cd, Tcl_Interp*interp, int argc, const char **argv){ MD5Init(&ctx); for(;;){ int n; - n = fread(zBuf, 1, sizeof(zBuf), in); + n = (int)fread(zBuf, 1, sizeof(zBuf), in); if( n<=0 ) break; MD5Update(&ctx, (unsigned char*)zBuf, (unsigned)n); } @@ -3520,7 +3516,7 @@ static void md5step(sqlite3_context *context, int argc, sqlite3_value **argv){ for(i=0; i<argc; i++){ const char *zData = (char*)sqlite3_value_text(argv[i]); if( zData ){ - MD5Update(p, (unsigned char*)zData, strlen(zData)); + MD5Update(p, (unsigned char*)zData, (int)strlen(zData)); } } } diff --git a/lib/libsqlite3/src/test1.c b/lib/libsqlite3/src/test1.c index e1220cf11f0..6849b5c7167 100644 --- a/lib/libsqlite3/src/test1.c +++ b/lib/libsqlite3/src/test1.c @@ -800,7 +800,7 @@ struct dstr { ** Append text to a dstr */ static void dstrAppend(struct dstr *p, const char *z, int divider){ - int n = strlen(z); + int n = (int)strlen(z); if( p->nUsed + n + 2 > p->nAlloc ){ char *zNew; p->nAlloc = p->nAlloc*2 + n + 200; @@ -3263,7 +3263,7 @@ static int test_bind_text16( char *value; int rc; - void (*xDel)() = (objc==6?SQLITE_STATIC:SQLITE_TRANSIENT); + void (*xDel)(void*) = (objc==6?SQLITE_STATIC:SQLITE_TRANSIENT); Tcl_Obj *oStmt = objv[objc-4]; Tcl_Obj *oN = objv[objc-3]; Tcl_Obj *oString = objv[objc-2]; @@ -3609,10 +3609,10 @@ static int test_prepare( if( sqlite3TestErrCode(interp, db, rc) ) return TCL_ERROR; if( zTail && objc>=5 ){ if( bytes>=0 ){ - bytes = bytes - (zTail-zSql); + bytes = bytes - (int)(zTail-zSql); } - if( strlen(zTail)<bytes ){ - bytes = strlen(zTail); + if( (int)strlen(zTail)<bytes ){ + bytes = (int)strlen(zTail); } Tcl_ObjSetVar2(interp, objv[4], 0, Tcl_NewStringObj(zTail, bytes), 0); } @@ -3667,7 +3667,7 @@ static int test_prepare_v2( if( sqlite3TestErrCode(interp, db, rc) ) return TCL_ERROR; if( zTail && objc>=5 ){ if( bytes>=0 ){ - bytes = bytes - (zTail-zSql); + bytes = bytes - (int)(zTail-zSql); } Tcl_ObjSetVar2(interp, objv[4], 0, Tcl_NewStringObj(zTail, bytes), 0); } @@ -3768,7 +3768,7 @@ static int test_prepare16( if( objc>=5 ){ if( zTail ){ - objlen = objlen - ((u8 *)zTail-(u8 *)zSql); + objlen = objlen - (int)((u8 *)zTail-(u8 *)zSql); }else{ objlen = 0; } @@ -3828,7 +3828,7 @@ static int test_prepare16_v2( if( objc>=5 ){ if( zTail ){ - objlen = objlen - ((u8 *)zTail-(u8 *)zSql); + objlen = objlen - (int)((u8 *)zTail-(u8 *)zSql); }else{ objlen = 0; } @@ -5816,6 +5816,7 @@ struct win32FileLocker { #if SQLITE_OS_WIN +#include <process.h> /* ** The background thread that does file locking. */ @@ -6128,7 +6129,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ #endif #ifdef SQLITE_ENABLE_COLUMN_METADATA {"sqlite3_column_database_name16", - test_stmt_utf16, sqlite3_column_database_name16}, + test_stmt_utf16, (void*)sqlite3_column_database_name16}, {"sqlite3_column_table_name16", test_stmt_utf16, (void*)sqlite3_column_table_name16}, {"sqlite3_column_origin_name16", test_stmt_utf16, (void*)sqlite3_column_origin_name16}, #endif diff --git a/lib/libsqlite3/src/test2.c b/lib/libsqlite3/src/test2.c index 8343692f6ab..8acdf6f4efe 100644 --- a/lib/libsqlite3/src/test2.c +++ b/lib/libsqlite3/src/test2.c @@ -547,7 +547,7 @@ static int fake_big_file( if( Tcl_GetInt(interp, argv[1], &n) ) return TCL_ERROR; pVfs = sqlite3_vfs_find(0); - nFile = strlen(argv[2]); + nFile = (int)strlen(argv[2]); zFile = sqlite3_malloc( nFile+2 ); if( zFile==0 ) return TCL_ERROR; memcpy(zFile, argv[2], nFile+1); diff --git a/lib/libsqlite3/src/test3.c b/lib/libsqlite3/src/test3.c index 9bac2cac13c..e460c42e46c 100644 --- a/lib/libsqlite3/src/test3.c +++ b/lib/libsqlite3/src/test3.c @@ -80,7 +80,7 @@ static int btree_open( sDb.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); sqlite3_mutex_enter(sDb.mutex); } - n = strlen(argv[1]); + n = (int)strlen(argv[1]); zFilename = sqlite3_malloc( n+2 ); if( zFilename==0 ) return TCL_ERROR; memcpy(zFilename, argv[1], n+1); @@ -465,7 +465,7 @@ static int btree_varint_test( if( Tcl_GetInt(interp, argv[4], (int*)&incr) ) return TCL_ERROR; in = start; in *= mult; - for(i=0; i<count; i++){ + for(i=0; i<(int)count; i++){ char zErr[200]; n1 = putVarint(zBuf, in); if( n1>9 || n1<1 ){ diff --git a/lib/libsqlite3/src/test6.c b/lib/libsqlite3/src/test6.c index 5f64cacca00..bae6b65dc25 100644 --- a/lib/libsqlite3/src/test6.c +++ b/lib/libsqlite3/src/test6.c @@ -177,7 +177,7 @@ static int writeDbFile(CrashFile *p, u8 *z, i64 iAmt, i64 iOff){ iSkip = 512; } if( (iAmt-iSkip)>0 ){ - rc = sqlite3OsWrite(p->pRealFile, &z[iSkip], iAmt-iSkip, iOff+iSkip); + rc = sqlite3OsWrite(p->pRealFile, &z[iSkip], (int)(iAmt-iSkip), iOff+iSkip); } return rc; } @@ -306,8 +306,8 @@ static int writeListSync(CrashFile *pFile, int isCrash){ } case 3: { /* Trash sectors */ u8 *zGarbage; - int iFirst = (pWrite->iOffset/g.iSectorSize); - int iLast = (pWrite->iOffset+pWrite->nBuf-1)/g.iSectorSize; + int iFirst = (int)(pWrite->iOffset/g.iSectorSize); + int iLast = (int)((pWrite->iOffset+pWrite->nBuf-1)/g.iSectorSize); assert(pWrite->zBuf); @@ -430,7 +430,7 @@ static int cfWrite( ){ CrashFile *pCrash = (CrashFile *)pFile; if( iAmt+iOfst>pCrash->iSize ){ - pCrash->iSize = iAmt+iOfst; + pCrash->iSize = (int)(iAmt+iOfst); } while( pCrash->iSize>pCrash->nData ){ u8 *zNew; @@ -454,7 +454,7 @@ static int cfTruncate(sqlite3_file *pFile, sqlite_int64 size){ CrashFile *pCrash = (CrashFile *)pFile; assert(size>=0); if( pCrash->iSize>size ){ - pCrash->iSize = size; + pCrash->iSize = (int)size; } return writeListAppend(pFile, size, 0, 0); } @@ -468,8 +468,8 @@ static int cfSync(sqlite3_file *pFile, int flags){ const char *zName = pCrash->zName; const char *zCrashFile = g.zCrashFile; - int nName = strlen(zName); - int nCrashFile = strlen(zCrashFile); + int nName = (int)strlen(zName); + int nCrashFile = (int)strlen(zCrashFile); if( nCrashFile>0 && zCrashFile[nCrashFile-1]=='*' ){ nCrashFile--; @@ -518,7 +518,7 @@ static int cfFileControl(sqlite3_file *pFile, int op, void *pArg){ i64 nByte = *(i64 *)pArg; if( nByte>pCrash->iSize ){ if( SQLITE_OK==writeListAppend(pFile, nByte, 0, 0) ){ - pCrash->iSize = nByte; + pCrash->iSize = (int)nByte; } } return SQLITE_OK; @@ -635,11 +635,11 @@ static int cfOpen( iChunk = PENDING_BYTE; } memset(pWrapper->zData, 0, pWrapper->nData); - rc = sqlite3OsRead(pReal, pWrapper->zData, iChunk, 0); + rc = sqlite3OsRead(pReal, pWrapper->zData, (int)iChunk, 0); if( SQLITE_OK==rc && pWrapper->iSize>(PENDING_BYTE+512) && isDb ){ i64 iOff = PENDING_BYTE+512; iChunk = pWrapper->iSize - iOff; - rc = sqlite3OsRead(pReal, &pWrapper->zData[iOff], iChunk, iOff); + rc = sqlite3OsRead(pReal, &pWrapper->zData[iOff], (int)iChunk, iOff); } }else{ rc = SQLITE_NOMEM; diff --git a/lib/libsqlite3/src/test8.c b/lib/libsqlite3/src/test8.c index 80e7adaf2c2..ba7e37372c7 100644 --- a/lib/libsqlite3/src/test8.c +++ b/lib/libsqlite3/src/test8.c @@ -192,7 +192,7 @@ static int getColumnNames( rc = SQLITE_NOMEM; goto out; } - nBytes += strlen(zName)+1; + nBytes += (int)strlen(zName)+1; } aCol = (char **)sqlite3MallocZero(nBytes); if( !aCol ){ @@ -1217,7 +1217,7 @@ static int echoRename(sqlite3_vtab *vtab, const char *zNewName){ } if( p->isPattern ){ - int nThis = strlen(p->zThis); + int nThis = (int)strlen(p->zThis); char *zSql = sqlite3_mprintf("ALTER TABLE %s RENAME TO %s%s", p->zTableName, zNewName, &p->zTableName[nThis] ); diff --git a/lib/libsqlite3/src/test_config.c b/lib/libsqlite3/src/test_config.c index 18442a49e06..f096ebf2362 100644 --- a/lib/libsqlite3/src/test_config.c +++ b/lib/libsqlite3/src/test_config.c @@ -420,6 +420,12 @@ Tcl_SetVar2(interp, "sqlite_options", "long_double", Tcl_SetVar2(interp, "sqlite_options", "rtree", "0", TCL_GLOBAL_ONLY); #endif +#ifdef SQLITE_RTREE_INT_ONLY + Tcl_SetVar2(interp, "sqlite_options", "rtree_int_only", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "rtree_int_only", "0", TCL_GLOBAL_ONLY); +#endif + #ifdef SQLITE_OMIT_SCHEMA_PRAGMAS Tcl_SetVar2(interp, "sqlite_options", "schema_pragmas", "0", TCL_GLOBAL_ONLY); #else diff --git a/lib/libsqlite3/src/test_func.c b/lib/libsqlite3/src/test_func.c index fff070e7eec..c4fe351cb9d 100644 --- a/lib/libsqlite3/src/test_func.c +++ b/lib/libsqlite3/src/test_func.c @@ -202,7 +202,7 @@ static void test_auxdata( }else { zRet[i*2] = '0'; } - n = strlen(z) + 1; + n = (int)strlen(z) + 1; zAux = testContextMalloc(pCtx, n); if( zAux ){ memcpy(zAux, z, n); diff --git a/lib/libsqlite3/src/test_fuzzer.c b/lib/libsqlite3/src/test_fuzzer.c index 60d56ee1ea8..10496f2ea74 100644 --- a/lib/libsqlite3/src/test_fuzzer.c +++ b/lib/libsqlite3/src/test_fuzzer.c @@ -308,8 +308,8 @@ static int fuzzerLoadOneRule( if( zFrom==0 ) zFrom = ""; if( zTo==0 ) zTo = ""; - nFrom = strlen(zFrom); - nTo = strlen(zTo); + nFrom = (int)strlen(zFrom); + nTo = (int)strlen(zTo); /* Silently ignore null transformations */ if( strcmp(zFrom, zTo)==0 ){ @@ -448,7 +448,7 @@ static char *fuzzerDequote(const char *zIn){ int nIn; /* Size of input string, in bytes */ char *zOut; /* Output (dequoted) string */ - nIn = strlen(zIn); + nIn = (int)strlen(zIn); zOut = sqlite3_malloc(nIn+1); if( zOut ){ char q = zIn[0]; /* Quote character (if any ) */ @@ -465,7 +465,7 @@ static char *fuzzerDequote(const char *zIn){ zOut[iOut++] = zIn[iIn]; } } - assert( strlen(zOut)<=nIn ); + assert( (int)strlen(zOut)<=nIn ); } return zOut; } @@ -513,7 +513,7 @@ static int fuzzerConnect( }else{ int nModule; /* Length of zModule, in bytes */ - nModule = strlen(zModule); + nModule = (int)strlen(zModule); pNew = sqlite3_malloc( sizeof(*pNew) + nModule + 1); if( pNew==0 ){ rc = SQLITE_NOMEM; @@ -870,11 +870,11 @@ static fuzzer_stem *fuzzerNewStem( fuzzer_rule *pRule; unsigned int h; - pNew = sqlite3_malloc( sizeof(*pNew) + strlen(zWord) + 1 ); + pNew = sqlite3_malloc( sizeof(*pNew) + (int)strlen(zWord) + 1 ); if( pNew==0 ) return 0; memset(pNew, 0, sizeof(*pNew)); pNew->zBasis = (char*)&pNew[1]; - pNew->nBasis = strlen(zWord); + pNew->nBasis = (int)strlen(zWord); memcpy(pNew->zBasis, zWord, pNew->nBasis+1); pRule = pCur->pVtab->pRule; while( fuzzerSkipRule(pRule, pNew, pCur->iRuleset) ){ @@ -997,7 +997,7 @@ static int fuzzerFilter( /* If the query term is longer than FUZZER_MX_OUTPUT_LENGTH bytes, this ** query will return zero rows. */ - if( strlen(zWord)<FUZZER_MX_OUTPUT_LENGTH ){ + if( (int)strlen(zWord)<FUZZER_MX_OUTPUT_LENGTH ){ pCur->pStem = pStem = fuzzerNewStem(pCur, zWord, (fuzzer_cost)0); if( pStem==0 ) return SQLITE_NOMEM; pStem->pRule = &pCur->nullRule; @@ -1127,8 +1127,7 @@ static int fuzzerBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ } /* -** A virtual table module that provides read-only access to a -** Tcl global variable namespace. +** A virtual table module that implements the "fuzzer". */ static sqlite3_module fuzzerModule = { 0, /* iVersion */ diff --git a/lib/libsqlite3/src/test_hexio.c b/lib/libsqlite3/src/test_hexio.c index e3258e869eb..b20b5ce730a 100644 --- a/lib/libsqlite3/src/test_hexio.c +++ b/lib/libsqlite3/src/test_hexio.c @@ -126,7 +126,7 @@ static int hexio_read( return TCL_ERROR; } fseek(in, offset, SEEK_SET); - got = fread(zBuf, 1, amt, in); + got = (int)fread(zBuf, 1, amt, in); fclose(in); if( got<0 ){ got = 0; @@ -178,7 +178,7 @@ static int hexio_write( return TCL_ERROR; } fseek(out, offset, SEEK_SET); - written = fwrite(aOut, 1, nOut, out); + written = (int)fwrite(aOut, 1, nOut, out); sqlite3_free(aOut); fclose(out); Tcl_SetObjResult(interp, Tcl_NewIntObj(written)); diff --git a/lib/libsqlite3/src/test_journal.c b/lib/libsqlite3/src/test_journal.c index 00567db598f..e8701a4eead 100644 --- a/lib/libsqlite3/src/test_journal.c +++ b/lib/libsqlite3/src/test_journal.c @@ -290,9 +290,9 @@ static jt_file *locateDatabaseHandle(const char *zJournal){ jt_file *pMain = 0; enterJtMutex(); for(pMain=g.pList; pMain; pMain=pMain->pNext){ - int nName = strlen(zJournal) - strlen("-journal"); + int nName = (int)(strlen(zJournal) - strlen("-journal")); if( (pMain->flags&SQLITE_OPEN_MAIN_DB) - && (strlen(pMain->zName)==nName) + && ((int)strlen(pMain->zName)==nName) && 0==memcmp(pMain->zName, zJournal, nName) && (pMain->eLock>=SQLITE_LOCK_RESERVED) ){ @@ -404,7 +404,7 @@ static int openTransaction(jt_file *pMain, jt_file *pJournal){ /* Calculate and store a checksum for each page in the database file. */ if( rc==SQLITE_OK ){ int ii; - for(ii=0; rc==SQLITE_OK && ii<pMain->nPage; ii++){ + for(ii=0; rc==SQLITE_OK && ii<(int)pMain->nPage; ii++){ i64 iOff = (i64)(pMain->nPagesize) * (i64)ii; if( iOff==PENDING_BYTE ) continue; rc = sqlite3OsRead(pMain->pReal, aData, pMain->nPagesize, iOff); @@ -466,7 +466,7 @@ static int readJournalFile(jt_file *p, jt_file *pMain){ continue; } } - nRec = (iSize-iOff) / (pMain->nPagesize+8); + nRec = (u32)((iSize-iOff) / (pMain->nPagesize+8)); } /* Read all the records that follow the journal-header just read. */ @@ -538,7 +538,7 @@ static int jtWrite( } if( p->flags&SQLITE_OPEN_MAIN_DB && p->pWritable ){ - if( iAmt<p->nPagesize + if( iAmt<(int)p->nPagesize && p->nPagesize%iAmt==0 && iOfst>=(PENDING_BYTE+512) && iOfst+iAmt<=PENDING_BYTE+p->nPagesize @@ -549,7 +549,7 @@ static int jtWrite( ** pending-byte page. */ }else{ - u32 pgno = iOfst/p->nPagesize + 1; + u32 pgno = (u32)(iOfst/p->nPagesize + 1); assert( (iAmt==1||iAmt==p->nPagesize) && ((iOfst+iAmt)%p->nPagesize)==0 ); assert( pgno<=p->nPage || p->nSync>0 ); assert( pgno>p->nPage || sqlite3BitvecTest(p->pWritable, pgno) ); @@ -578,7 +578,7 @@ static int jtTruncate(sqlite3_file *pFile, sqlite_int64 size){ if( p->flags&SQLITE_OPEN_MAIN_DB && p->pWritable ){ u32 pgno; u32 locking_page = (u32)(PENDING_BYTE/p->nPagesize+1); - for(pgno=size/p->nPagesize+1; pgno<=p->nPage; pgno++){ + for(pgno=(u32)(size/p->nPagesize+1); pgno<=p->nPage; pgno++){ assert( pgno==locking_page || sqlite3BitvecTest(p->pWritable, pgno) ); } } @@ -723,7 +723,7 @@ static int jtOpen( ** returning. */ static int jtDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ - int nPath = strlen(zPath); + int nPath = (int)strlen(zPath); if( nPath>8 && 0==strcmp("-journal", &zPath[nPath-8]) ){ /* Deleting a journal file. The end of a transaction. */ jt_file *pMain = locateDatabaseHandle(zPath); diff --git a/lib/libsqlite3/src/test_malloc.c b/lib/libsqlite3/src/test_malloc.c index 037dfd140a1..09b8f738e84 100644 --- a/lib/libsqlite3/src/test_malloc.c +++ b/lib/libsqlite3/src/test_malloc.c @@ -1323,7 +1323,8 @@ static int test_db_status( { "LOOKASIDE_MISS_SIZE", SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE }, { "LOOKASIDE_MISS_FULL", SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL }, { "CACHE_HIT", SQLITE_DBSTATUS_CACHE_HIT }, - { "CACHE_MISS", SQLITE_DBSTATUS_CACHE_MISS } + { "CACHE_MISS", SQLITE_DBSTATUS_CACHE_MISS }, + { "CACHE_WRITE", SQLITE_DBSTATUS_CACHE_WRITE } }; Tcl_Obj *pResult; if( objc!=4 ){ diff --git a/lib/libsqlite3/src/test_multiplex.c b/lib/libsqlite3/src/test_multiplex.c index e2b6720652f..a3b3e2f2712 100644 --- a/lib/libsqlite3/src/test_multiplex.c +++ b/lib/libsqlite3/src/test_multiplex.c @@ -329,6 +329,7 @@ static sqlite3_file *multiplexSubOpen( ** database may therefore not grow to larger than 400 chunks. Attempting ** to open chunk 401 indicates the database is full. */ if( iChunk>=SQLITE_MULTIPLEX_JOURNAL_8_3_OFFSET ){ + sqlite3_log(SQLITE_FULL, "multiplexed chunk overflow: %s", pGroup->zName); *rc = SQLITE_FULL; return 0; } @@ -347,7 +348,13 @@ static sqlite3_file *multiplexSubOpen( }else{ *rc = pOrigVfs->xAccess(pOrigVfs, pGroup->aReal[iChunk].z, SQLITE_ACCESS_EXISTS, &bExists); - if( *rc || !bExists ) return 0; + if( *rc || !bExists ){ + if( *rc ){ + sqlite3_log(*rc, "multiplexor.xAccess failure on %s", + pGroup->aReal[iChunk].z); + } + return 0; + } flags &= ~SQLITE_OPEN_CREATE; } pSubOpen = sqlite3_malloc( pOrigVfs->szOsFile ); @@ -359,6 +366,8 @@ static sqlite3_file *multiplexSubOpen( *rc = pOrigVfs->xOpen(pOrigVfs, pGroup->aReal[iChunk].z, pSubOpen, flags, pOutFlags); if( (*rc)!=SQLITE_OK ){ + sqlite3_log(*rc, "multiplexor.xOpen failure on %s", + pGroup->aReal[iChunk].z); sqlite3_free(pSubOpen); pGroup->aReal[iChunk].p = 0; return 0; @@ -529,7 +538,7 @@ static int multiplexOpen( pGroup->bEnabled = -1; pGroup->bTruncate = sqlite3_uri_boolean(zUri, "truncate", (flags & SQLITE_OPEN_MAIN_DB)==0); - pGroup->szChunk = sqlite3_uri_int64(zUri, "chunksize", + pGroup->szChunk = (int)sqlite3_uri_int64(zUri, "chunksize", SQLITE_MULTIPLEX_CHUNK_SIZE); pGroup->szChunk = (pGroup->szChunk+0xffff)&~0xffff; if( zName ){ @@ -597,7 +606,7 @@ static int multiplexOpen( bExists = multiplexSubSize(pGroup, 1, &rc)>0; if( rc==SQLITE_OK && bExists && sz==(sz&0xffff0000) && sz>0 && sz!=pGroup->szChunk ){ - pGroup->szChunk = sz; + pGroup->szChunk = (int)sz; }else if( rc==SQLITE_OK && !bExists && sz>pGroup->szChunk ){ pGroup->bEnabled = 0; } @@ -641,7 +650,7 @@ static int multiplexDelete( /* If the main chunk was deleted successfully, also delete any subsequent ** chunks - starting with the last (highest numbered). */ - int nName = strlen(zName); + int nName = (int)strlen(zName); char *z; z = sqlite3_malloc(nName + 5); if( z==0 ){ diff --git a/lib/libsqlite3/src/test_onefile.c b/lib/libsqlite3/src/test_onefile.c index cd7db008cff..69867441b8c 100644 --- a/lib/libsqlite3/src/test_onefile.c +++ b/lib/libsqlite3/src/test_onefile.c @@ -288,7 +288,7 @@ static int tmpWrite( ){ tmp_file *pTmp = (tmp_file *)pFile; if( (iAmt+iOfst)>pTmp->nAlloc ){ - int nNew = 2*(iAmt+iOfst+pTmp->nAlloc); + int nNew = (int)(2*(iAmt+iOfst+pTmp->nAlloc)); char *zNew = sqlite3_realloc(pTmp->zAlloc, nNew); if( !zNew ){ return SQLITE_NOMEM; @@ -297,7 +297,7 @@ static int tmpWrite( pTmp->nAlloc = nNew; } memcpy(&pTmp->zAlloc[iOfst], zBuf, iAmt); - pTmp->nSize = MAX(pTmp->nSize, iOfst+iAmt); + pTmp->nSize = (int)MAX(pTmp->nSize, iOfst+iAmt); return SQLITE_OK; } @@ -306,7 +306,7 @@ static int tmpWrite( */ static int tmpTruncate(sqlite3_file *pFile, sqlite_int64 size){ tmp_file *pTmp = (tmp_file *)pFile; - pTmp->nSize = MIN(pTmp->nSize, size); + pTmp->nSize = (int)MIN(pTmp->nSize, size); return SQLITE_OK; } @@ -418,7 +418,7 @@ static int fsRead( /* Journal file. */ int iRem = iAmt; int iBuf = 0; - int ii = iOfst; + int ii = (int)iOfst; while( iRem>0 && rc==SQLITE_OK ){ int iRealOff = pReal->nBlob - BLOCKSIZE*((ii/BLOCKSIZE)+1) + ii%BLOCKSIZE; int iRealAmt = MIN(iRem, BLOCKSIZE - (iRealOff%BLOCKSIZE)); @@ -453,14 +453,14 @@ static int fsWrite( }else{ rc = pF->pMethods->xWrite(pF, zBuf, iAmt, iOfst+BLOCKSIZE); if( rc==SQLITE_OK ){ - pReal->nDatabase = MAX(pReal->nDatabase, iAmt+iOfst); + pReal->nDatabase = (int)MAX(pReal->nDatabase, iAmt+iOfst); } } }else{ /* Journal file. */ int iRem = iAmt; int iBuf = 0; - int ii = iOfst; + int ii = (int)iOfst; while( iRem>0 && rc==SQLITE_OK ){ int iRealOff = pReal->nBlob - BLOCKSIZE*((ii/BLOCKSIZE)+1) + ii%BLOCKSIZE; int iRealAmt = MIN(iRem, BLOCKSIZE - (iRealOff%BLOCKSIZE)); @@ -475,7 +475,7 @@ static int fsWrite( } } if( rc==SQLITE_OK ){ - pReal->nJournal = MAX(pReal->nJournal, iAmt+iOfst); + pReal->nJournal = (int)MAX(pReal->nJournal, iAmt+iOfst); } } @@ -489,9 +489,9 @@ static int fsTruncate(sqlite3_file *pFile, sqlite_int64 size){ fs_file *p = (fs_file *)pFile; fs_real_file *pReal = p->pReal; if( p->eType==DATABASE_FILE ){ - pReal->nDatabase = MIN(pReal->nDatabase, size); + pReal->nDatabase = (int)MIN(pReal->nDatabase, size); }else{ - pReal->nJournal = MIN(pReal->nJournal, size); + pReal->nJournal = (int)MIN(pReal->nJournal, size); } return SQLITE_OK; } @@ -606,7 +606,7 @@ static int fsOpen( p->eType = eType; assert(strlen("-journal")==8); - nName = strlen(zName)-((eType==JOURNAL_FILE)?8:0); + nName = (int)strlen(zName)-((eType==JOURNAL_FILE)?8:0); pReal=pFsVfs->pFileList; for(; pReal && strncmp(pReal->zName, zName, nName); pReal=pReal->pNext); @@ -641,7 +641,7 @@ static int fsOpen( pReal->nBlob = BLOBSIZE; }else{ unsigned char zS[4]; - pReal->nBlob = size; + pReal->nBlob = (int)size; rc = pRealFile->pMethods->xRead(pRealFile, zS, 4, 0); pReal->nDatabase = (zS[0]<<24)+(zS[1]<<16)+(zS[2]<<8)+zS[3]; if( rc==SQLITE_OK ){ @@ -687,7 +687,7 @@ static int fsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ fs_vfs_t *pFsVfs = (fs_vfs_t *)pVfs; fs_real_file *pReal; sqlite3_file *pF; - int nName = strlen(zPath) - 8; + int nName = (int)strlen(zPath) - 8; assert(strlen("-journal")==8); assert(strcmp("-journal", &zPath[nName])==0); @@ -717,7 +717,7 @@ static int fsAccess( fs_vfs_t *pFsVfs = (fs_vfs_t *)pVfs; fs_real_file *pReal; int isJournal = 0; - int nName = strlen(zPath); + int nName = (int)strlen(zPath); if( flags!=SQLITE_ACCESS_EXISTS ){ sqlite3_vfs *pParent = ((fs_vfs_t *)pVfs)->pParent; diff --git a/lib/libsqlite3/src/test_osinst.c b/lib/libsqlite3/src/test_osinst.c index b7f350577c8..531433313ee 100644 --- a/lib/libsqlite3/src/test_osinst.c +++ b/lib/libsqlite3/src/test_osinst.c @@ -242,7 +242,7 @@ static sqlite3_uint64 vfslog_time(){ } #endif -static void vfslog_call(sqlite3_vfs *, int, int, int, int, int, int); +static void vfslog_call(sqlite3_vfs *, int, int, sqlite3_int64, int, int, int); static void vfslog_string(sqlite3_vfs *, const char *); /* @@ -648,7 +648,7 @@ static void vfslog_call( sqlite3_vfs *pVfs, int eEvent, int iFileid, - int nClick, + sqlite3_int64 nClick, int return_code, int size, int offset @@ -661,7 +661,7 @@ static void vfslog_call( zRec = (unsigned char *)&p->aBuf[p->nBuf]; put32bits(&zRec[0], eEvent); put32bits(&zRec[4], iFileid); - put32bits(&zRec[8], nClick); + put32bits(&zRec[8], (unsigned int)(nClick&0xffff)); put32bits(&zRec[12], return_code); put32bits(&zRec[16], size); put32bits(&zRec[20], offset); @@ -671,7 +671,7 @@ static void vfslog_call( static void vfslog_string(sqlite3_vfs *pVfs, const char *zStr){ VfslogVfs *p = (VfslogVfs *)pVfs; unsigned char *zRec; - int nStr = zStr ? strlen(zStr) : 0; + int nStr = zStr ? (int)strlen(zStr) : 0; if( (4+nStr+p->nBuf)>sizeof(p->aBuf) ){ vfslog_flush(p); } @@ -720,7 +720,7 @@ int sqlite3_vfslog_new( return SQLITE_ERROR; } - nVfs = strlen(zVfs); + nVfs = (int)strlen(zVfs); nByte = sizeof(VfslogVfs) + pParent->szOsFile + nVfs+1+pParent->mxPathname+1; p = (VfslogVfs *)sqlite3_malloc(nByte); memset(p, 0, nByte); @@ -1043,7 +1043,7 @@ static int vlogColumn( } case 1: { char *zStr = pCsr->zTransient; - if( val!=0 && val<pCsr->nFile ){ + if( val!=0 && val<(unsigned)pCsr->nFile ){ zStr = pCsr->azFile[val]; } sqlite3_result_text(ctx, zStr, -1, SQLITE_TRANSIENT); diff --git a/lib/libsqlite3/src/test_quota.c b/lib/libsqlite3/src/test_quota.c index c8ed7389e54..87bd1e91836 100644 --- a/lib/libsqlite3/src/test_quota.c +++ b/lib/libsqlite3/src/test_quota.c @@ -120,6 +120,9 @@ struct quota_FILE { FILE *f; /* Open stdio file pointer */ sqlite3_int64 iOfst; /* Current offset into the file */ quotaFile *pFile; /* The file record in the quota system */ +#if SQLITE_OS_WIN + char *zMbcsName; /* Full MBCS pathname of the file */ +#endif }; @@ -341,7 +344,7 @@ static quotaFile *quotaFindFile( pFile = pFile->pNext; } if( pFile==0 && createFlag ){ - int nName = strlen(zName); + int nName = (int)(strlen(zName) & 0x3fffffff); pFile = (quotaFile *)sqlite3_malloc( sizeof(*pFile) + nName + 1 ); if( pFile ){ memset(pFile, 0, sizeof(*pFile)); @@ -419,7 +422,7 @@ static quotaFile *quotaFindFile( */ static char *quota_utf8_to_mbcs(const char *zUtf8){ #if SQLITE_OS_WIN - int n; /* Bytes in zUtf8 */ + size_t n; /* Bytes in zUtf8 */ int nWide; /* number of UTF-16 characters */ int nMbcs; /* Bytes of MBCS */ LPWSTR zTmpWide; /* The UTF16 text */ @@ -897,7 +900,7 @@ int sqlite3_quota_set( pGroup = pGroup->pNext; } if( pGroup==0 ){ - int nPattern = strlen(zPattern); + int nPattern = (int)(strlen(zPattern) & 0x3fffffff); if( iLimit<=0 ){ quotaLeave(); return SQLITE_OK; @@ -979,7 +982,7 @@ int sqlite3_quota_file(const char *zFilename){ quota_FILE *sqlite3_quota_fopen(const char *zFilename, const char *zMode){ quota_FILE *p = 0; char *zFull = 0; - char *zFullTranslated; + char *zFullTranslated = 0; int rc; quotaGroup *pGroup; quotaFile *pFile; @@ -995,7 +998,6 @@ quota_FILE *sqlite3_quota_fopen(const char *zFilename, const char *zMode){ zFullTranslated = quota_utf8_to_mbcs(zFull); if( zFullTranslated==0 ) goto quota_fopen_error; p->f = fopen(zFullTranslated, zMode); - quota_mbcs_free(zFullTranslated); if( p->f==0 ) goto quota_fopen_error; quotaEnter(); pGroup = quotaGroupFind(zFull); @@ -1010,9 +1012,13 @@ quota_FILE *sqlite3_quota_fopen(const char *zFilename, const char *zMode){ } quotaLeave(); sqlite3_free(zFull); +#if SQLITE_OS_WIN + p->zMbcsName = zFullTranslated; +#endif return p; quota_fopen_error: + quota_mbcs_free(zFullTranslated); sqlite3_free(zFull); if( p && p->f ) fclose(p->f); sqlite3_free(p); @@ -1045,6 +1051,7 @@ size_t sqlite3_quota_fwrite( sqlite3_int64 iEnd; sqlite3_int64 szNew; quotaFile *pFile; + size_t rc; iOfst = ftell(p->f); iEnd = iOfst + size*nmemb; @@ -1060,7 +1067,7 @@ size_t sqlite3_quota_fwrite( } if( szNew>pGroup->iLimit && pGroup->iLimit>0 ){ iEnd = pGroup->iLimit - pGroup->iSize + pFile->iSize; - nmemb = (iEnd - iOfst)/size; + nmemb = (size_t)((iEnd - iOfst)/size); iEnd = iOfst + size*nmemb; szNew = pGroup->iSize - pFile->iSize + iEnd; } @@ -1068,8 +1075,23 @@ size_t sqlite3_quota_fwrite( pGroup->iSize = szNew; pFile->iSize = iEnd; quotaLeave(); + }else{ + pFile = 0; + } + rc = fwrite(pBuf, size, nmemb, p->f); + + /* If the write was incomplete, adjust the file size and group size + ** downward */ + if( rc<nmemb && pFile ){ + size_t nWritten = rc>=0 ? rc : 0; + sqlite3_int64 iNewEnd = iOfst + size*nWritten; + if( iNewEnd<iEnd ) iNewEnd = iEnd; + quotaEnter(); + pFile->pGroup->iSize += iNewEnd - pFile->iSize; + pFile->iSize = iNewEnd; + quotaLeave(); } - return fwrite(pBuf, size, nmemb, p->f); + return rc; } /* @@ -1093,6 +1115,9 @@ int sqlite3_quota_fclose(quota_FILE *p){ } quotaLeave(); } +#if SQLITE_OS_WIN + quota_mbcs_free(p->zMbcsName); +#endif sqlite3_free(p); return rc; } @@ -1136,11 +1161,88 @@ long sqlite3_quota_ftell(quota_FILE *p){ } /* +** Truncate a file to szNew bytes. +*/ +int sqlite3_quota_ftruncate(quota_FILE *p, sqlite3_int64 szNew){ + quotaFile *pFile = p->pFile; + int rc; + if( (pFile = p->pFile)!=0 && pFile->iSize<szNew ){ + quotaGroup *pGroup; + if( pFile->iSize<szNew ){ + /* This routine cannot be used to extend a file that is under + ** quota management. Only true truncation is allowed. */ + return -1; + } + pGroup = pFile->pGroup; + quotaEnter(); + pGroup->iSize += szNew - pFile->iSize; + quotaLeave(); + } +#if SQLITE_OS_UNIX + rc = ftruncate(fileno(p->f), szNew); +#endif +#if SQLITE_OS_WIN + rc = _chsize_s(_fileno(p->f), szNew); +#endif + if( pFile && rc==0 ){ + quotaGroup *pGroup = pFile->pGroup; + quotaEnter(); + pGroup->iSize += szNew - pFile->iSize; + pFile->iSize = szNew; + quotaLeave(); + } + return rc; +} + +/* +** Determine the time that the given file was last modified, in +** seconds size 1970. Write the result into *pTime. Return 0 on +** success and non-zero on any kind of error. +*/ +int sqlite3_quota_file_mtime(quota_FILE *p, time_t *pTime){ + int rc; +#if SQLITE_OS_UNIX + struct stat buf; + rc = fstat(fileno(p->f), &buf); +#endif +#if SQLITE_OS_WIN + struct _stati64 buf; + rc = _stati64(p->zMbcsName, &buf); +#endif + if( rc==0 ) *pTime = buf.st_mtime; + return rc; +} + +/* +** Return the true size of the file, as reported by the operating +** system. +*/ +sqlite3_int64 sqlite3_quota_file_truesize(quota_FILE *p){ + int rc; +#if SQLITE_OS_UNIX + struct stat buf; + rc = fstat(fileno(p->f), &buf); +#endif +#if SQLITE_OS_WIN + struct _stati64 buf; + rc = _stati64(p->zMbcsName, &buf); +#endif + return rc==0 ? buf.st_size : -1; +} + +/* +** Return the size of the file, as it is known to the quota subsystem. +*/ +sqlite3_int64 sqlite3_quota_file_size(quota_FILE *p){ + return p->pFile ? p->pFile->iSize : -1; +} + +/* ** Remove a managed file. Update quotas accordingly. */ int sqlite3_quota_remove(const char *zFilename){ char *zFull; /* Full pathname for zFilename */ - int nFull; /* Number of bytes in zFilename */ + size_t nFull; /* Number of bytes in zFilename */ int rc; /* Result code */ quotaGroup *pGroup; /* Group containing zFilename */ quotaFile *pFile; /* A file in the group */ @@ -1480,7 +1582,7 @@ static int test_quota_fread( char *zBuf; int sz; int nElem; - int got; + size_t got; if( objc!=4 ){ Tcl_WrongNumArgs(interp, 1, objv, "HANDLE SIZE NELEM"); @@ -1515,7 +1617,7 @@ static int test_quota_fwrite( char *zBuf; int sz; int nElem; - int got; + size_t got; if( objc!=5 ){ Tcl_WrongNumArgs(interp, 1, objv, "HANDLE SIZE NELEM CONTENT"); @@ -1526,7 +1628,7 @@ static int test_quota_fwrite( if( Tcl_GetIntFromObj(interp, objv[3], &nElem) ) return TCL_ERROR; zBuf = Tcl_GetString(objv[4]); got = sqlite3_quota_fwrite(zBuf, sz, nElem, p); - Tcl_SetObjResult(interp, Tcl_NewIntObj(got)); + Tcl_SetObjResult(interp, Tcl_NewWideIntObj(got)); return TCL_OK; } @@ -1657,6 +1759,96 @@ static int test_quota_ftell( } /* +** tclcmd: sqlite3_quota_ftruncate HANDLE SIZE +*/ +static int test_quota_ftruncate( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + quota_FILE *p; + sqlite3_int64 x; + Tcl_WideInt w; + int rc; + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "HANDLE SIZE"); + return TCL_ERROR; + } + p = sqlite3TestTextToPtr(Tcl_GetString(objv[1])); + if( Tcl_GetWideIntFromObj(interp, objv[2], &w) ) return TCL_ERROR; + x = (sqlite3_int64)w; + rc = sqlite3_quota_ftruncate(p, x); + Tcl_SetObjResult(interp, Tcl_NewIntObj(rc)); + return TCL_OK; +} + +/* +** tclcmd: sqlite3_quota_file_size HANDLE +*/ +static int test_quota_file_size( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + quota_FILE *p; + sqlite3_int64 x; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "HANDLE"); + return TCL_ERROR; + } + p = sqlite3TestTextToPtr(Tcl_GetString(objv[1])); + x = sqlite3_quota_file_size(p); + Tcl_SetObjResult(interp, Tcl_NewWideIntObj(x)); + return TCL_OK; +} + +/* +** tclcmd: sqlite3_quota_file_truesize HANDLE +*/ +static int test_quota_file_truesize( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + quota_FILE *p; + sqlite3_int64 x; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "HANDLE"); + return TCL_ERROR; + } + p = sqlite3TestTextToPtr(Tcl_GetString(objv[1])); + x = sqlite3_quota_file_truesize(p); + Tcl_SetObjResult(interp, Tcl_NewWideIntObj(x)); + return TCL_OK; +} + +/* +** tclcmd: sqlite3_quota_file_mtime HANDLE +*/ +static int test_quota_file_mtime( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + quota_FILE *p; + time_t t; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "HANDLE"); + return TCL_ERROR; + } + p = sqlite3TestTextToPtr(Tcl_GetString(objv[1])); + t = 0; + sqlite3_quota_file_mtime(p, &t); + Tcl_SetObjResult(interp, Tcl_NewWideIntObj(t)); + return TCL_OK; +} + + +/* ** tclcmd: sqlite3_quota_remove FILENAME */ static int test_quota_remove( @@ -1713,21 +1905,25 @@ int Sqlitequota_Init(Tcl_Interp *interp){ char *zName; Tcl_ObjCmdProc *xProc; } aCmd[] = { - { "sqlite3_quota_initialize", test_quota_initialize }, - { "sqlite3_quota_shutdown", test_quota_shutdown }, - { "sqlite3_quota_set", test_quota_set }, - { "sqlite3_quota_file", test_quota_file }, - { "sqlite3_quota_dump", test_quota_dump }, - { "sqlite3_quota_fopen", test_quota_fopen }, - { "sqlite3_quota_fread", test_quota_fread }, - { "sqlite3_quota_fwrite", test_quota_fwrite }, - { "sqlite3_quota_fclose", test_quota_fclose }, - { "sqlite3_quota_fflush", test_quota_fflush }, - { "sqlite3_quota_fseek", test_quota_fseek }, - { "sqlite3_quota_rewind", test_quota_rewind }, - { "sqlite3_quota_ftell", test_quota_ftell }, - { "sqlite3_quota_remove", test_quota_remove }, - { "sqlite3_quota_glob", test_quota_glob }, + { "sqlite3_quota_initialize", test_quota_initialize }, + { "sqlite3_quota_shutdown", test_quota_shutdown }, + { "sqlite3_quota_set", test_quota_set }, + { "sqlite3_quota_file", test_quota_file }, + { "sqlite3_quota_dump", test_quota_dump }, + { "sqlite3_quota_fopen", test_quota_fopen }, + { "sqlite3_quota_fread", test_quota_fread }, + { "sqlite3_quota_fwrite", test_quota_fwrite }, + { "sqlite3_quota_fclose", test_quota_fclose }, + { "sqlite3_quota_fflush", test_quota_fflush }, + { "sqlite3_quota_fseek", test_quota_fseek }, + { "sqlite3_quota_rewind", test_quota_rewind }, + { "sqlite3_quota_ftell", test_quota_ftell }, + { "sqlite3_quota_ftruncate", test_quota_ftruncate }, + { "sqlite3_quota_file_size", test_quota_file_size }, + { "sqlite3_quota_file_truesize", test_quota_file_truesize }, + { "sqlite3_quota_file_mtime", test_quota_file_mtime }, + { "sqlite3_quota_remove", test_quota_remove }, + { "sqlite3_quota_glob", test_quota_glob }, }; int i; diff --git a/lib/libsqlite3/src/test_quota.h b/lib/libsqlite3/src/test_quota.h index a2fddbbc439..9bd4312c6c0 100644 --- a/lib/libsqlite3/src/test_quota.h +++ b/lib/libsqlite3/src/test_quota.h @@ -29,6 +29,14 @@ #ifndef _QUOTA_H_ #include "sqlite3.h" #include <stdio.h> +#include <sys/types.h> +#include <sys/stat.h> +#if SQLITE_OS_UNIX +# include <unistd.h> +#endif +#if SQLITE_OS_WIN +# include <windows.h> +#endif /* Make this callable from C++ */ #ifdef __cplusplus @@ -183,6 +191,48 @@ void sqlite3_quota_rewind(quota_FILE*); long sqlite3_quota_ftell(quota_FILE*); /* +** Truncate a file previously opened by sqlite3_quota_fopen(). Return +** zero on success and non-zero on any kind of failure. +** +** The newSize argument must be less than or equal to the current file size. +** Any attempt to "truncate" a file to a larger size results in +** undefined behavior. +*/ +int sqlite3_quota_ftrunate(quota_FILE*, sqlite3_int64 newSize); + +/* +** Return the last modification time of the opened file, in seconds +** since 1970. +*/ +int sqlite3_quota_file_mtime(quota_FILE*, time_t *pTime); + +/* +** Return the size of the file as it is known to the quota system. +** +** This size might be different from the true size of the file on +** disk if some outside process has modified the file without using the +** quota mechanism, or if calls to sqlite3_quota_fwrite() have occurred +** which have increased the file size, but those writes have not yet been +** forced to disk using sqlite3_quota_fflush(). +** +** Return -1 if the file is not participating in quota management. +*/ +sqlite3_int64 sqlite3_quota_file_size(quota_FILE*); + +/* +** Return the true size of the file. +** +** The true size should be the same as the size of the file as known +** to the quota system, however the sizes might be different if the +** file has been extended or truncated via some outside process or if +** pending writes have not yet been flushed to disk. +** +** Return -1 if the file does not exist or if the size of the file +** cannot be determined for some reason. +*/ +sqlite3_int64 sqlite3_quota_file_truesize(quota_FILE*); + +/* ** Delete a file from the disk, if that file is under quota management. ** Adjust quotas accordingly. ** diff --git a/lib/libsqlite3/src/test_rtree.c b/lib/libsqlite3/src/test_rtree.c index 9745b005417..d3c9e0cb3de 100644 --- a/lib/libsqlite3/src/test_rtree.c +++ b/lib/libsqlite3/src/test_rtree.c @@ -49,7 +49,11 @@ static void circle_del(void *p){ static int circle_geom( sqlite3_rtree_geometry *p, int nCoord, +#ifdef SQLITE_RTREE_INT_ONLY + sqlite3_int64 *aCoord, +#else double *aCoord, +#endif int *pRes ){ int i; /* Iterator variable */ @@ -188,8 +192,12 @@ static int gHere = 42; */ static int cube_geom( sqlite3_rtree_geometry *p, - int nCoord, + int nCoord, +#ifdef SQLITE_RTREE_INT_ONLY + sqlite3_int64 *aCoord, +#else double *aCoord, +#endif int *piRes ){ Cube *pCube = (Cube *)p->pUser; diff --git a/lib/libsqlite3/src/test_spellfix.c b/lib/libsqlite3/src/test_spellfix.c new file mode 100644 index 00000000000..5a221e0b1b0 --- /dev/null +++ b/lib/libsqlite3/src/test_spellfix.c @@ -0,0 +1,1951 @@ +/* +** 2012 April 10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This module implements a VIRTUAL TABLE that can be used to search +** a large vocabulary for close matches. For example, this virtual +** table can be used to suggest corrections to misspelled words. Or, +** it could be used with FTS4 to do full-text search using potentially +** misspelled words. +** +** Create an instance of the virtual table this way: +** +** CREATE VIRTUAL TABLE demo USING spellfix1; +** +** The "spellfix1" term is the name of this module. The "demo" is the +** name of the virtual table you will be creating. The table is initially +** empty. You have to populate it with your vocabulary. Suppose you +** have a list of words in a table named "big_vocabulary". Then do this: +** +** INSERT INTO demo(word) SELECT word FROM big_vocabulary; +** +** If you intend to use this virtual table in cooperation with an FTS4 +** table (for spelling correctly of search terms) then you can extract +** the vocabulary using an fts3aux table: +** +** INSERT INTO demo(word) SELECT term FROM search_aux WHERE col='*'; +** +** You can also provide the virtual table with a "rank" for each word. +** The "rank" is an estimate of how common the word is. Larger numbers +** mean the word is more common. If you omit the rank when populating +** the table, then a rank of 1 is assumed. But if you have rank +** information, you can supply it and the virtual table will show a +** slight preference for selecting more commonly used terms. To +** populate the rank from an fts4aux table "search_aux" do something +** like this: +** +** INSERT INTO demo(word,rank) +** SELECT term, documents FROM search_aux WHERE col='*'; +** +** To query the virtual table, include a MATCH operator in the WHERE +** clause. For example: +** +** SELECT word FROM demo WHERE word MATCH 'kennasaw'; +** +** Using a dataset of American place names (derived from +** http://geonames.usgs.gov/domestic/download_data.htm) the query above +** returns 20 results beginning with: +** +** kennesaw +** kenosha +** kenesaw +** kenaga +** keanak +** +** If you append the character '*' to the end of the pattern, then +** a prefix search is performed. For example: +** +** SELECT word FROM demo WHERE word MATCH 'kennes*'; +** +** Yields 20 results beginning with: +** +** kennesaw +** kennestone +** kenneson +** kenneys +** keanes +** keenes +** +** The virtual table actually has a unique rowid with five columns plus three +** extra hidden columns. The columns are as follows: +** +** rowid A unique integer number associated with each +** vocabulary item in the table. This can be used +** as a foreign key on other tables in the database. +** +** word The text of the word that matches the pattern. +** Both word and pattern can contains unicode characters +** and can be mixed case. +** +** rank This is the rank of the word, as specified in the +** original INSERT statement. +** +** distance This is an edit distance or Levensthein distance going +** from the pattern to the word. +** +** langid This is the language-id of the word. All queries are +** against a single language-id, which defaults to 0. +** For any given query this value is the same on all rows. +** +** score The score is a combination of rank and distance. The +** idea is that a lower score is better. The virtual table +** attempts to find words with the lowest score and +** by default (unless overridden by ORDER BY) returns +** results in order of increasing score. +** +** top (HIDDEN) For any query, this value is the same on all +** rows. It is an integer which is the maximum number of +** rows that will be output. The actually number of rows +** output might be less than this number, but it will never +** be greater. The default value for top is 20, but that +** can be changed for each query by including a term of +** the form "top=N" in the WHERE clause of the query. +** +** scope (HIDDEN) For any query, this value is the same on all +** rows. The scope is a measure of how widely the virtual +** table looks for matching words. Smaller values of +** scope cause a broader search. The scope is normally +** choosen automatically and is capped at 4. Applications +** can change the scope by including a term of the form +** "scope=N" in the WHERE clause of the query. Increasing +** the scope will make the query run faster, but will reduce +** the possible corrections. +** +** srchcnt (HIDDEN) For any query, this value is the same on all +** rows. This value is an integer which is the number of +** of words examined using the edit-distance algorithm to +** find the top matches that are ultimately displayed. This +** value is for diagnostic use only. +** +** soundslike (HIDDEN) When inserting vocabulary entries, this field +** can be set to an spelling that matches what the word +** sounds like. See the DEALING WITH UNUSUAL AND DIFFICULT +** SPELLINGS section below for details. +** +** When inserting into or updating the virtual table, only the rowid, word, +** rank, and langid may be changes. Any attempt to set or modify the values +** of distance, score, top, scope, or srchcnt is silently ignored. +** +** ALGORITHM +** +** A shadow table named "%_vocab" (where the % is replaced by the name of +** the virtual table; Ex: "demo_vocab" for the "demo" virtual table) is +** constructed with these columns: +** +** id The unique id (INTEGER PRIMARY KEY) +** +** rank The rank of word. +** +** langid The language id for this entry. +** +** word The original UTF8 text of the vocabulary word +** +** k1 The word transliterated into lower-case ASCII. +** There is a standard table of mappings from non-ASCII +** characters into ASCII. Examples: "æ" -> "ae", +** "þ" -> "th", "ß" -> "ss", "á" -> "a", ... The +** accessory function spellfix1_translit(X) will do +** the non-ASCII to ASCII mapping. The built-in lower(X) +** function will convert to lower-case. Thus: +** k1 = lower(spellfix1_translit(word)). +** +** k2 This field holds a phonetic code derived from k1. Letters +** that have similar sounds are mapped into the same symbol. +** For example, all vowels and vowel clusters become the +** single symbol "A". And the letters "p", "b", "f", and +** "v" all become "B". All nasal sounds are represented +** as "N". And so forth. The mapping is base on +** ideas found in Soundex, Metaphone, and other +** long-standing phonetic matching systems. This key can +** be generated by the function spellfix1_charclass(X). +** Hence: k2 = spellfix1_charclass(k1) +** +** There is also a function for computing the Wagner edit distance or the +** Levenshtein distance between a pattern and a word. This function +** is exposed as spellfix1_editdist(X,Y). The edit distance function +** returns the "cost" of converting X into Y. Some transformations +** cost more than others. Changing one vowel into a different vowel, +** for example is relatively cheap, as is doubling a constant, or +** omitting the second character of a double-constant. Other transformations +** or more expensive. The idea is that the edit distance function returns +** a low cost of words that are similar and a higher cost for words +** that are futher apart. In this implementation, the maximum cost +** of any single-character edit (delete, insert, or substitute) is 100, +** with lower costs for some edits (such as transforming vowels). +** +** The "score" for a comparison is the edit distance between the pattern +** and the word, adjusted down by the base-2 logorithm of the word rank. +** For example, a match with distance 100 but rank 1000 would have a +** score of 122 (= 100 - log2(1000) + 32) where as a match with distance +** 100 with a rank of 1 would have a score of 131 (100 - log2(1) + 32). +** (NB: The constant 32 is added to each score to keep it from going +** negative in case the edit distance is zero.) In this way, frequently +** used words get a slightly lower cost which tends to move them toward +** the top of the list of alternative spellings. +** +** A straightforward implementation of a spelling corrector would be +** to compare the search term against every word in the vocabulary +** and select the 20 with the lowest scores. However, there will +** typically be hundreds of thousands or millions of words in the +** vocabulary, and so this approach is not fast enough. +** +** Suppose the term that is being spell-corrected is X. To limit +** the search space, X is converted to a k2-like key using the +** equivalent of: +** +** key = spellfix1_charclass(lower(spellfix1_translit(X))) +** +** This key is then limited to "scope" characters. The default scope +** value is 4, but an alternative scope can be specified using the +** "scope=N" term in the WHERE clause. After the key has been truncated, +** the edit distance is run against every term in the vocabulary that +** has a k2 value that begins with the abbreviated key. +** +** For example, suppose the input word is "Paskagula". The phonetic +** key is "BACACALA" which is then truncated to 4 characters "BACA". +** The edit distance is then run on the 4980 entries (out of +** 272,597 entries total) of the vocabulary whose k2 values begin with +** BACA, yielding "Pascagoula" as the best match. +** +** Only terms of the vocabulary with a matching langid are searched. +** Hence, the same table can contain entries from multiple languages +** and only the requested language will be used. The default langid +** is 0. +** +** DEALING WITH UNUSUAL AND DIFFICULT SPELLINGS +** +** The algorithm above works quite well for most cases, but there are +** exceptions. These exceptions can be dealt with by making additional +** entries in the virtual table using the "soundslike" column. +** +** For example, many words of Greek origin begin with letters "ps" where +** the "p" is silent. Ex: psalm, pseudonym, psoriasis, psyche. In +** another example, many Scottish surnames can be spelled with an +** initial "Mac" or "Mc". Thus, "MacKay" and "McKay" are both pronounced +** the same. +** +** Accommodation can be made for words that are not spelled as they +** sound by making additional entries into the virtual table for the +** same word, but adding an alternative spelling in the "soundslike" +** column. For example, the canonical entry for "psalm" would be this: +** +** INSERT INTO demo(word) VALUES('psalm'); +** +** To enhance the ability to correct the spelling of "salm" into +** "psalm", make an addition entry like this: +** +** INSERT INTO demo(word,soundslike) VALUES('psalm','salm'); +** +** It is ok to make multiple entries for the same word as long as +** each entry has a different soundslike value. Note that if no +** soundslike value is specified, the soundslike defaults to the word +** itself. +** +** Listed below are some cases where it might make sense to add additional +** soundslike entries. The specific entries will depend on the application +** and the target language. +** +** * Silent "p" in words beginning with "ps": psalm, psyche +** +** * Silent "p" in words beginning with "pn": pneumonia, pneumatic +** +** * Silent "p" in words beginning with "pt": pterodactyl, ptolemaic +** +** * Silent "d" in words beginning with "dj": djinn, Djikarta +** +** * Silent "k" in words beginning with "kn": knight, Knuthson +** +** * Silent "g" in words beginning with "gn": gnarly, gnome, gnat +** +** * "Mac" versus "Mc" beginning Scottish surnames +** +** * "Tch" sounds in Slavic words: Tchaikovsky vs. Chaykovsky +** +** * The letter "j" pronounced like "h" in Spanish: LaJolla +** +** * Words beginning with "wr" versus "r": write vs. rite +** +** * Miscellanous problem words such as "debt", "tsetse", +** "Nguyen", "Van Nuyes". +*/ +#if SQLITE_CORE +# include "sqliteInt.h" +#else +# include <string.h> +# include <stdio.h> +# include <stdlib.h> +# include "sqlite3ext.h" + SQLITE_EXTENSION_INIT1 +#endif /* !SQLITE_CORE */ + +/* +** Character classes for ASCII characters: +** +** 0 '' Silent letters: H W +** 1 'A' Any vowel: A E I O U (Y) +** 2 'B' A bilabeal stop or fricative: B F P V +** 3 'C' Other fricatives or back stops: C G J K Q S X Z +** 4 'D' Alveolar stops: D T +** 5 'H' Letter H at the beginning of a word +** 6 'L' Glides: L R +** 7 'M' Nasals: M N +** 8 'W' Letter W at the beginning of a word +** 9 'Y' Letter Y at the beginning of a word. +** 10 '9' A digit: 0 1 2 3 4 5 6 7 8 9 +** 11 ' ' White space +** 12 '?' Other. +*/ +#define CCLASS_SILENT 0 +#define CCLASS_VOWEL 1 +#define CCLASS_B 2 +#define CCLASS_C 3 +#define CCLASS_D 4 +#define CCLASS_H 5 +#define CCLASS_L 6 +#define CCLASS_M 7 +#define CCLASS_W 8 +#define CCLASS_Y 9 +#define CCLASS_DIGIT 10 +#define CCLASS_SPACE 11 +#define CCLASS_OTHER 12 + +/* +** The following table gives the character class for non-initial ASCII +** characters. +*/ +static const unsigned char midClass[] = { + /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ + /* 0x */ 12, 12, 12, 12, 12, 12, 12, 12, 12, 11, 11, 12, 11, 12, 12, 12, + /* 1x */ 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, + /* 2x */ 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, + /* 3x */ 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 12, 12, 12, 12, 12, 12, + /* 4x */ 12, 1, 2, 3, 4, 1, 2, 3, 0, 1, 3, 3, 6, 7, 7, 1, + /* 5x */ 2, 3, 6, 3, 4, 1, 2, 0, 3, 1, 3, 12, 12, 12, 12, 12, + /* 6x */ 12, 1, 2, 3, 4, 1, 2, 3, 0, 1, 3, 3, 6, 7, 7, 1, + /* 7x */ 2, 3, 6, 3, 4, 1, 2, 0, 3, 1, 3, 12, 12, 12, 12, 12, +}; + +/* +** This tables gives the character class for ASCII characters that form the +** initial character of a word. The only difference from midClass is with +** the letters H, W, and Y. +*/ +static const unsigned char initClass[] = { + /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ + /* 0x */ 12, 12, 12, 12, 12, 12, 12, 12, 12, 11, 11, 12, 11, 12, 12, 12, + /* 1x */ 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, + /* 2x */ 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, + /* 3x */ 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 12, 12, 12, 12, 12, 12, + /* 4x */ 12, 1, 2, 3, 4, 1, 2, 3, 5, 1, 3, 3, 6, 7, 7, 1, + /* 5x */ 2, 3, 6, 3, 4, 1, 2, 8, 3, 9, 3, 12, 12, 12, 12, 12, + /* 6x */ 12, 1, 2, 3, 4, 1, 2, 3, 5, 1, 3, 3, 6, 7, 7, 1, + /* 7x */ 2, 3, 6, 3, 4, 1, 2, 8, 3, 9, 3, 12, 12, 12, 12, 12, +}; + +/* +** Mapping from the character class number (0-12) to a symbol for each +** character class. Note that initClass[] can be used to map the class +** symbol back into the class number. +*/ +static const unsigned char className[] = ".ABCDHLMWY9 ?"; + +/* +** Generate a string of character classes corresponding to the +** ASCII characters in the input string zIn. If the input is not +** ASCII then the behavior is undefined. +** +** Space to hold the result is obtained from sqlite3_malloc() +** +** Return NULL if memory allocation fails. +*/ +static unsigned char *characterClassString(const unsigned char *zIn, int nIn){ + unsigned char *zOut = sqlite3_malloc( nIn + 1 ); + int i; + int nOut = 0; + char cPrev = 0x77; + const unsigned char *aClass = initClass; + + if( zOut==0 ) return 0; + for(i=0; i<nIn; i++){ + unsigned char c = zIn[i]; + c = aClass[c&0x7f]; + if( c==CCLASS_OTHER && cPrev!=CCLASS_DIGIT ) continue; + cPrev = c; + if( c==CCLASS_SILENT ) continue; + if( c==CCLASS_SPACE ) continue; + aClass = midClass; + c = className[c]; + if( c!=zOut[nOut-1] ) zOut[nOut++] = c; + } + zOut[nOut] = 0; + return zOut; +} + +/* +** This is an SQL function wrapper around characterClassString(). See +** the description of characterClassString() for additional information. +*/ +static void characterClassSqlFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zIn; + unsigned char *zOut; + + zIn = sqlite3_value_text(argv[0]); + if( zIn==0 ) return; + zOut = characterClassString(zIn, sqlite3_value_bytes(argv[0])); + if( zOut==0 ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_result_text(context, (char*)zOut, -1, sqlite3_free); + } +} + +/* +** Return the character class number for a character given its +** context. +*/ +static char characterClass(char cPrev, char c){ + return cPrev==0 ? initClass[c&0x7f] : midClass[c&0x7f]; +} + +/* +** Return the cost of inserting or deleting character c immediately +** following character cPrev. If cPrev==0, that means c is the first +** character of the word. +*/ +static int insertOrDeleteCost(char cPrev, char c){ + char classC = characterClass(cPrev, c); + char classCprev; + + if( classC==CCLASS_SILENT ){ + /* Insert or delete "silent" characters such as H or W */ + return 1; + } + if( cPrev==c ){ + /* Repeated characters, or miss a repeat */ + return 10; + } + classCprev = characterClass(cPrev, cPrev); + if( classC==classCprev ){ + if( classC==CCLASS_VOWEL ){ + /* Remove or add a new vowel to a vowel cluster */ + return 15; + }else{ + /* Remove or add a consonant not in the same class */ + return 50; + } + } + + /* any other character insertion or deletion */ + return 100; +} + +/* +** Divide the insertion cost by this factor when appending to the +** end of the word. +*/ +#define FINAL_INS_COST_DIV 4 + +/* +** Return the cost of substituting cTo in place of cFrom assuming +** the previous character is cPrev. If cPrev==0 then cTo is the first +** character of the word. +*/ +static int substituteCost(char cPrev, char cFrom, char cTo){ + char classFrom, classTo; + if( cFrom==cTo ){ + /* Exact match */ + return 0; + } + if( cFrom==(cTo^0x20) && ((cTo>='A' && cTo<='Z') || (cTo>='a' && cTo<='z')) ){ + /* differ only in case */ + return 0; + } + classFrom = characterClass(cPrev, cFrom); + classTo = characterClass(cPrev, cTo); + if( classFrom==classTo ){ + /* Same character class */ + return classFrom=='A' ? 25 : 40; + } + if( classFrom>=CCLASS_B && classFrom<=CCLASS_Y + && classTo>=CCLASS_B && classTo<=CCLASS_Y ){ + /* Convert from one consonant to another, but in a different class */ + return 75; + } + /* Any other subsitution */ + return 100; +} + +/* +** Given two strings zA and zB which are pure ASCII, return the cost +** of transforming zA into zB. If zA ends with '*' assume that it is +** a prefix of zB and give only minimal penalty for extra characters +** on the end of zB. +** +** Smaller numbers mean a closer match. +** +** Negative values indicate an error: +** -1 One of the inputs is NULL +** -2 Non-ASCII characters on input +** -3 Unable to allocate memory +*/ +static int editdist(const char *zA, const char *zB){ + int nA, nB; /* Number of characters in zA[] and zB[] */ + int xA, xB; /* Loop counters for zA[] and zB[] */ + char cA, cB; /* Current character of zA and zB */ + char cAprev, cBprev; /* Previous character of zA and zB */ + int d; /* North-west cost value */ + int dc = 0; /* North-west character value */ + int res; /* Final result */ + int *m; /* The cost matrix */ + char *cx; /* Corresponding character values */ + int *toFree = 0; /* Malloced space */ + int mStack[60+15]; /* Stack space to use if not too much is needed */ + + /* Early out if either input is NULL */ + if( zA==0 || zB==0 ) return -1; + + /* Skip any common prefix */ + while( zA[0] && zA[0]==zB[0] ){ dc = zA[0]; zA++; zB++; } + if( zA[0]==0 && zB[0]==0 ) return 0; + +#if 0 + printf("A=\"%s\" B=\"%s\" dc=%c\n", zA, zB, dc?dc:' '); +#endif + + /* Verify input strings and measure their lengths */ + for(nA=0; zA[nA]; nA++){ + if( zA[nA]>127 ) return -2; + } + for(nB=0; zB[nB]; nB++){ + if( zB[nB]>127 ) return -2; + } + + /* Special processing if either string is empty */ + if( nA==0 ){ + cBprev = dc; + for(xB=res=0; (cB = zB[xB])!=0; xB++){ + res += insertOrDeleteCost(cBprev, cB)/FINAL_INS_COST_DIV; + cBprev = cB; + } + return res; + } + if( nB==0 ){ + cAprev = dc; + for(xA=res=0; (cA = zA[xA])!=0; xA++){ + res += insertOrDeleteCost(cAprev, cA); + cAprev = cA; + } + return res; + } + + /* A is a prefix of B */ + if( zA[0]=='*' && zA[1]==0 ) return 0; + + /* Allocate and initialize the Wagner matrix */ + if( nB<(sizeof(mStack)*4)/(sizeof(mStack[0])*5) ){ + m = mStack; + }else{ + m = toFree = sqlite3_malloc( (nB+1)*5*sizeof(m[0])/4 ); + if( m==0 ) return -3; + } + cx = (char*)&m[nB+1]; + + /* Compute the Wagner edit distance */ + m[0] = 0; + cx[0] = dc; + cBprev = dc; + for(xB=1; xB<=nB; xB++){ + cB = zB[xB-1]; + cx[xB] = cB; + m[xB] = m[xB-1] + insertOrDeleteCost(cBprev, cB); + cBprev = cB; + } + cAprev = dc; + for(xA=1; xA<=nA; xA++){ + int lastA = (xA==nA); + cA = zA[xA-1]; + if( cA=='*' && lastA ) break; + d = m[0]; + dc = cx[0]; + m[0] = d + insertOrDeleteCost(cAprev, cA); + cBprev = 0; + for(xB=1; xB<=nB; xB++){ + int totalCost, insCost, delCost, subCost, ncx; + cB = zB[xB-1]; + + /* Cost to insert cB */ + insCost = insertOrDeleteCost(cx[xB-1], cB); + if( lastA ) insCost /= FINAL_INS_COST_DIV; + + /* Cost to delete cA */ + delCost = insertOrDeleteCost(cx[xB], cA); + + /* Cost to substitute cA->cB */ + subCost = substituteCost(cx[xB-1], cA, cB); + + /* Best cost */ + totalCost = insCost + m[xB-1]; + ncx = cB; + if( (delCost + m[xB])<totalCost ){ + totalCost = delCost + m[xB]; + ncx = cA; + } + if( (subCost + d)<totalCost ){ + totalCost = subCost + d; + } + +#if 0 + printf("%d,%d d=%4d u=%4d r=%4d dc=%c cA=%c cB=%c" + " ins=%4d del=%4d sub=%4d t=%4d ncx=%c\n", + xA, xB, d, m[xB], m[xB-1], dc?dc:' ', cA, cB, + insCost, delCost, subCost, totalCost, ncx?ncx:' '); +#endif + + /* Update the matrix */ + d = m[xB]; + dc = cx[xB]; + m[xB] = totalCost; + cx[xB] = ncx; + cBprev = cB; + } + cAprev = cA; + } + + /* Free the wagner matrix and return the result */ + if( cA=='*' && nB>nA ){ + res = m[nA]; + for(xB=nA+1; xB<=nB; xB++){ + if( m[xB]<res ) res = m[xB]; + } + }else{ + res = m[nB]; + } + sqlite3_free(toFree); + return res; +} + +/* +** Function: editdist(A,B) +** +** Return the cost of transforming string A into string B. Both strings +** must be pure ASCII text. If A ends with '*' then it is assumed to be +** a prefix of B and extra characters on the end of B have minimal additional +** cost. +*/ +static void editdistSqlFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int res = editdist((const char*)sqlite3_value_text(argv[0]), + (const char*)sqlite3_value_text(argv[1])); + if( res<0 ){ + if( res==(-3) ){ + sqlite3_result_error_nomem(context); + }else if( res==(-2) ){ + sqlite3_result_error(context, "non-ASCII input to editdist()", -1); + }else{ + sqlite3_result_error(context, "NULL input to editdist()", -1); + } + }else{ + sqlite3_result_int(context, res); + } +} + +#if !SQLITE_CORE +/* +** This lookup table is used to help decode the first byte of +** a multi-byte UTF8 character. +*/ +static const unsigned char sqlite3Utf8Trans1[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, +}; +#endif + +/* +** Return the value of the first UTF-8 character in the string. +*/ +static int utf8Read(const unsigned char *z, int n, int *pSize){ + int c, i; + + if( n==0 ){ + c = i = 0; + }else{ + c = z[0]; + i = 1; + if( c>=0xc0 ){ + c = sqlite3Utf8Trans1[c-0xc0]; + while( i<n && (z[i] & 0xc0)==0x80 ){ + c = (c<<6) + (0x3f & z[i++]); + } + } + } + *pSize = i; + return c; +} + +/* +** Table of translations from unicode characters into ASCII. +*/ +static const struct { + unsigned short int cFrom; + unsigned char cTo0, cTo1; +} translit[] = { + { 0x00A0, 0x20, 0x00 }, /* to */ + { 0x00B5, 0x75, 0x00 }, /* µ to u */ + { 0x00C0, 0x41, 0x00 }, /* À to A */ + { 0x00C1, 0x41, 0x00 }, /* Á to A */ + { 0x00C2, 0x41, 0x00 }, /*  to A */ + { 0x00C3, 0x41, 0x00 }, /* à to A */ + { 0x00C4, 0x41, 0x65 }, /* Ä to Ae */ + { 0x00C5, 0x41, 0x61 }, /* Å to Aa */ + { 0x00C6, 0x41, 0x45 }, /* Æ to AE */ + { 0x00C7, 0x43, 0x00 }, /* Ç to C */ + { 0x00C8, 0x45, 0x00 }, /* È to E */ + { 0x00C9, 0x45, 0x00 }, /* É to E */ + { 0x00CA, 0x45, 0x00 }, /* Ê to E */ + { 0x00CB, 0x45, 0x00 }, /* Ë to E */ + { 0x00CC, 0x49, 0x00 }, /* Ì to I */ + { 0x00CD, 0x49, 0x00 }, /* Í to I */ + { 0x00CE, 0x49, 0x00 }, /* Î to I */ + { 0x00CF, 0x49, 0x00 }, /* Ï to I */ + { 0x00D0, 0x44, 0x00 }, /* Ð to D */ + { 0x00D1, 0x4E, 0x00 }, /* Ñ to N */ + { 0x00D2, 0x4F, 0x00 }, /* Ò to O */ + { 0x00D3, 0x4F, 0x00 }, /* Ó to O */ + { 0x00D4, 0x4F, 0x00 }, /* Ô to O */ + { 0x00D5, 0x4F, 0x00 }, /* Õ to O */ + { 0x00D6, 0x4F, 0x65 }, /* Ö to Oe */ + { 0x00D7, 0x78, 0x00 }, /* × to x */ + { 0x00D8, 0x4F, 0x00 }, /* Ø to O */ + { 0x00D9, 0x55, 0x00 }, /* Ù to U */ + { 0x00DA, 0x55, 0x00 }, /* Ú to U */ + { 0x00DB, 0x55, 0x00 }, /* Û to U */ + { 0x00DC, 0x55, 0x65 }, /* Ü to Ue */ + { 0x00DD, 0x59, 0x00 }, /* Ý to Y */ + { 0x00DE, 0x54, 0x68 }, /* Þ to Th */ + { 0x00DF, 0x73, 0x73 }, /* ß to ss */ + { 0x00E0, 0x61, 0x00 }, /* à to a */ + { 0x00E1, 0x61, 0x00 }, /* á to a */ + { 0x00E2, 0x61, 0x00 }, /* â to a */ + { 0x00E3, 0x61, 0x00 }, /* ã to a */ + { 0x00E4, 0x61, 0x65 }, /* ä to ae */ + { 0x00E5, 0x61, 0x61 }, /* å to aa */ + { 0x00E6, 0x61, 0x65 }, /* æ to ae */ + { 0x00E7, 0x63, 0x00 }, /* ç to c */ + { 0x00E8, 0x65, 0x00 }, /* è to e */ + { 0x00E9, 0x65, 0x00 }, /* é to e */ + { 0x00EA, 0x65, 0x00 }, /* ê to e */ + { 0x00EB, 0x65, 0x00 }, /* ë to e */ + { 0x00EC, 0x69, 0x00 }, /* ì to i */ + { 0x00ED, 0x69, 0x00 }, /* í to i */ + { 0x00EE, 0x69, 0x00 }, /* î to i */ + { 0x00EF, 0x69, 0x00 }, /* ï to i */ + { 0x00F0, 0x64, 0x00 }, /* ð to d */ + { 0x00F1, 0x6E, 0x00 }, /* ñ to n */ + { 0x00F2, 0x6F, 0x00 }, /* ò to o */ + { 0x00F3, 0x6F, 0x00 }, /* ó to o */ + { 0x00F4, 0x6F, 0x00 }, /* ô to o */ + { 0x00F5, 0x6F, 0x00 }, /* õ to o */ + { 0x00F6, 0x6F, 0x65 }, /* ö to oe */ + { 0x00F7, 0x3A, 0x00 }, /* ÷ to : */ + { 0x00F8, 0x6F, 0x00 }, /* ø to o */ + { 0x00F9, 0x75, 0x00 }, /* ù to u */ + { 0x00FA, 0x75, 0x00 }, /* ú to u */ + { 0x00FB, 0x75, 0x00 }, /* û to u */ + { 0x00FC, 0x75, 0x65 }, /* ü to ue */ + { 0x00FD, 0x79, 0x00 }, /* ý to y */ + { 0x00FE, 0x74, 0x68 }, /* þ to th */ + { 0x00FF, 0x79, 0x00 }, /* ÿ to y */ + { 0x0100, 0x41, 0x00 }, /* Ā to A */ + { 0x0101, 0x61, 0x00 }, /* ā to a */ + { 0x0102, 0x41, 0x00 }, /* Ă to A */ + { 0x0103, 0x61, 0x00 }, /* ă to a */ + { 0x0104, 0x41, 0x00 }, /* Ą to A */ + { 0x0105, 0x61, 0x00 }, /* ą to a */ + { 0x0106, 0x43, 0x00 }, /* Ć to C */ + { 0x0107, 0x63, 0x00 }, /* ć to c */ + { 0x0108, 0x43, 0x68 }, /* Ĉ to Ch */ + { 0x0109, 0x63, 0x68 }, /* ĉ to ch */ + { 0x010A, 0x43, 0x00 }, /* Ċ to C */ + { 0x010B, 0x63, 0x00 }, /* ċ to c */ + { 0x010C, 0x43, 0x00 }, /* Č to C */ + { 0x010D, 0x63, 0x00 }, /* č to c */ + { 0x010E, 0x44, 0x00 }, /* Ď to D */ + { 0x010F, 0x64, 0x00 }, /* ď to d */ + { 0x0110, 0x44, 0x00 }, /* Đ to D */ + { 0x0111, 0x64, 0x00 }, /* đ to d */ + { 0x0112, 0x45, 0x00 }, /* Ē to E */ + { 0x0113, 0x65, 0x00 }, /* ē to e */ + { 0x0114, 0x45, 0x00 }, /* Ĕ to E */ + { 0x0115, 0x65, 0x00 }, /* ĕ to e */ + { 0x0116, 0x45, 0x00 }, /* Ė to E */ + { 0x0117, 0x65, 0x00 }, /* ė to e */ + { 0x0118, 0x45, 0x00 }, /* Ę to E */ + { 0x0119, 0x65, 0x00 }, /* ę to e */ + { 0x011A, 0x45, 0x00 }, /* Ě to E */ + { 0x011B, 0x65, 0x00 }, /* ě to e */ + { 0x011C, 0x47, 0x68 }, /* Ĝ to Gh */ + { 0x011D, 0x67, 0x68 }, /* ĝ to gh */ + { 0x011E, 0x47, 0x00 }, /* Ğ to G */ + { 0x011F, 0x67, 0x00 }, /* ğ to g */ + { 0x0120, 0x47, 0x00 }, /* Ġ to G */ + { 0x0121, 0x67, 0x00 }, /* ġ to g */ + { 0x0122, 0x47, 0x00 }, /* Ģ to G */ + { 0x0123, 0x67, 0x00 }, /* ģ to g */ + { 0x0124, 0x48, 0x68 }, /* Ĥ to Hh */ + { 0x0125, 0x68, 0x68 }, /* ĥ to hh */ + { 0x0126, 0x48, 0x00 }, /* Ħ to H */ + { 0x0127, 0x68, 0x00 }, /* ħ to h */ + { 0x0128, 0x49, 0x00 }, /* Ĩ to I */ + { 0x0129, 0x69, 0x00 }, /* ĩ to i */ + { 0x012A, 0x49, 0x00 }, /* Ī to I */ + { 0x012B, 0x69, 0x00 }, /* ī to i */ + { 0x012C, 0x49, 0x00 }, /* Ĭ to I */ + { 0x012D, 0x69, 0x00 }, /* ĭ to i */ + { 0x012E, 0x49, 0x00 }, /* Į to I */ + { 0x012F, 0x69, 0x00 }, /* į to i */ + { 0x0130, 0x49, 0x00 }, /* İ to I */ + { 0x0131, 0x69, 0x00 }, /* ı to i */ + { 0x0132, 0x49, 0x4A }, /* IJ to IJ */ + { 0x0133, 0x69, 0x6A }, /* ij to ij */ + { 0x0134, 0x4A, 0x68 }, /* Ĵ to Jh */ + { 0x0135, 0x6A, 0x68 }, /* ĵ to jh */ + { 0x0136, 0x4B, 0x00 }, /* Ķ to K */ + { 0x0137, 0x6B, 0x00 }, /* ķ to k */ + { 0x0138, 0x6B, 0x00 }, /* ĸ to k */ + { 0x0139, 0x4C, 0x00 }, /* Ĺ to L */ + { 0x013A, 0x6C, 0x00 }, /* ĺ to l */ + { 0x013B, 0x4C, 0x00 }, /* Ļ to L */ + { 0x013C, 0x6C, 0x00 }, /* ļ to l */ + { 0x013D, 0x4C, 0x00 }, /* Ľ to L */ + { 0x013E, 0x6C, 0x00 }, /* ľ to l */ + { 0x013F, 0x4C, 0x2E }, /* Ŀ to L. */ + { 0x0140, 0x6C, 0x2E }, /* ŀ to l. */ + { 0x0141, 0x4C, 0x00 }, /* Ł to L */ + { 0x0142, 0x6C, 0x00 }, /* ł to l */ + { 0x0143, 0x4E, 0x00 }, /* Ń to N */ + { 0x0144, 0x6E, 0x00 }, /* ń to n */ + { 0x0145, 0x4E, 0x00 }, /* Ņ to N */ + { 0x0146, 0x6E, 0x00 }, /* ņ to n */ + { 0x0147, 0x4E, 0x00 }, /* Ň to N */ + { 0x0148, 0x6E, 0x00 }, /* ň to n */ + { 0x0149, 0x27, 0x6E }, /* ʼn to 'n */ + { 0x014A, 0x4E, 0x47 }, /* Ŋ to NG */ + { 0x014B, 0x6E, 0x67 }, /* ŋ to ng */ + { 0x014C, 0x4F, 0x00 }, /* Ō to O */ + { 0x014D, 0x6F, 0x00 }, /* ō to o */ + { 0x014E, 0x4F, 0x00 }, /* Ŏ to O */ + { 0x014F, 0x6F, 0x00 }, /* ŏ to o */ + { 0x0150, 0x4F, 0x00 }, /* Ő to O */ + { 0x0151, 0x6F, 0x00 }, /* ő to o */ + { 0x0152, 0x4F, 0x45 }, /* Œ to OE */ + { 0x0153, 0x6F, 0x65 }, /* œ to oe */ + { 0x0154, 0x52, 0x00 }, /* Ŕ to R */ + { 0x0155, 0x72, 0x00 }, /* ŕ to r */ + { 0x0156, 0x52, 0x00 }, /* Ŗ to R */ + { 0x0157, 0x72, 0x00 }, /* ŗ to r */ + { 0x0158, 0x52, 0x00 }, /* Ř to R */ + { 0x0159, 0x72, 0x00 }, /* ř to r */ + { 0x015A, 0x53, 0x00 }, /* Ś to S */ + { 0x015B, 0x73, 0x00 }, /* ś to s */ + { 0x015C, 0x53, 0x68 }, /* Ŝ to Sh */ + { 0x015D, 0x73, 0x68 }, /* ŝ to sh */ + { 0x015E, 0x53, 0x00 }, /* Ş to S */ + { 0x015F, 0x73, 0x00 }, /* ş to s */ + { 0x0160, 0x53, 0x00 }, /* Š to S */ + { 0x0161, 0x73, 0x00 }, /* š to s */ + { 0x0162, 0x54, 0x00 }, /* Ţ to T */ + { 0x0163, 0x74, 0x00 }, /* ţ to t */ + { 0x0164, 0x54, 0x00 }, /* Ť to T */ + { 0x0165, 0x74, 0x00 }, /* ť to t */ + { 0x0166, 0x54, 0x00 }, /* Ŧ to T */ + { 0x0167, 0x74, 0x00 }, /* ŧ to t */ + { 0x0168, 0x55, 0x00 }, /* Ũ to U */ + { 0x0169, 0x75, 0x00 }, /* ũ to u */ + { 0x016A, 0x55, 0x00 }, /* Ū to U */ + { 0x016B, 0x75, 0x00 }, /* ū to u */ + { 0x016C, 0x55, 0x00 }, /* Ŭ to U */ + { 0x016D, 0x75, 0x00 }, /* ŭ to u */ + { 0x016E, 0x55, 0x00 }, /* Ů to U */ + { 0x016F, 0x75, 0x00 }, /* ů to u */ + { 0x0170, 0x55, 0x00 }, /* Ű to U */ + { 0x0171, 0x75, 0x00 }, /* ű to u */ + { 0x0172, 0x55, 0x00 }, /* Ų to U */ + { 0x0173, 0x75, 0x00 }, /* ų to u */ + { 0x0174, 0x57, 0x00 }, /* Ŵ to W */ + { 0x0175, 0x77, 0x00 }, /* ŵ to w */ + { 0x0176, 0x59, 0x00 }, /* Ŷ to Y */ + { 0x0177, 0x79, 0x00 }, /* ŷ to y */ + { 0x0178, 0x59, 0x00 }, /* Ÿ to Y */ + { 0x0179, 0x5A, 0x00 }, /* Ź to Z */ + { 0x017A, 0x7A, 0x00 }, /* ź to z */ + { 0x017B, 0x5A, 0x00 }, /* Ż to Z */ + { 0x017C, 0x7A, 0x00 }, /* ż to z */ + { 0x017D, 0x5A, 0x00 }, /* Ž to Z */ + { 0x017E, 0x7A, 0x00 }, /* ž to z */ + { 0x017F, 0x73, 0x00 }, /* ſ to s */ + { 0x0192, 0x66, 0x00 }, /* ƒ to f */ + { 0x0218, 0x53, 0x00 }, /* Ș to S */ + { 0x0219, 0x73, 0x00 }, /* ș to s */ + { 0x021A, 0x54, 0x00 }, /* Ț to T */ + { 0x021B, 0x74, 0x00 }, /* ț to t */ + { 0x0386, 0x41, 0x00 }, /* Ά to A */ + { 0x0388, 0x45, 0x00 }, /* Έ to E */ + { 0x0389, 0x49, 0x00 }, /* Ή to I */ + { 0x038A, 0x49, 0x00 }, /* Ί to I */ + { 0x038C, 0x4f, 0x00 }, /* Ό to O */ + { 0x038E, 0x59, 0x00 }, /* Ύ to Y */ + { 0x038F, 0x4f, 0x00 }, /* Ώ to O */ + { 0x0390, 0x69, 0x00 }, /* ΐ to i */ + { 0x0391, 0x41, 0x00 }, /* Α to A */ + { 0x0392, 0x42, 0x00 }, /* Β to B */ + { 0x0393, 0x47, 0x00 }, /* Γ to G */ + { 0x0394, 0x44, 0x00 }, /* Δ to D */ + { 0x0395, 0x45, 0x00 }, /* Ε to E */ + { 0x0396, 0x5a, 0x00 }, /* Ζ to Z */ + { 0x0397, 0x49, 0x00 }, /* Η to I */ + { 0x0398, 0x54, 0x68 }, /* Θ to Th */ + { 0x0399, 0x49, 0x00 }, /* Ι to I */ + { 0x039A, 0x4b, 0x00 }, /* Κ to K */ + { 0x039B, 0x4c, 0x00 }, /* Λ to L */ + { 0x039C, 0x4d, 0x00 }, /* Μ to M */ + { 0x039D, 0x4e, 0x00 }, /* Ν to N */ + { 0x039E, 0x58, 0x00 }, /* Ξ to X */ + { 0x039F, 0x4f, 0x00 }, /* Ο to O */ + { 0x03A0, 0x50, 0x00 }, /* Π to P */ + { 0x03A1, 0x52, 0x00 }, /* Ρ to R */ + { 0x03A3, 0x53, 0x00 }, /* Σ to S */ + { 0x03A4, 0x54, 0x00 }, /* Τ to T */ + { 0x03A5, 0x59, 0x00 }, /* Υ to Y */ + { 0x03A6, 0x46, 0x00 }, /* Φ to F */ + { 0x03A7, 0x43, 0x68 }, /* Χ to Ch */ + { 0x03A8, 0x50, 0x73 }, /* Ψ to Ps */ + { 0x03A9, 0x4f, 0x00 }, /* Ω to O */ + { 0x03AA, 0x49, 0x00 }, /* Ϊ to I */ + { 0x03AB, 0x59, 0x00 }, /* Ϋ to Y */ + { 0x03AC, 0x61, 0x00 }, /* ά to a */ + { 0x03AD, 0x65, 0x00 }, /* έ to e */ + { 0x03AE, 0x69, 0x00 }, /* ή to i */ + { 0x03AF, 0x69, 0x00 }, /* ί to i */ + { 0x03B1, 0x61, 0x00 }, /* α to a */ + { 0x03B2, 0x62, 0x00 }, /* β to b */ + { 0x03B3, 0x67, 0x00 }, /* γ to g */ + { 0x03B4, 0x64, 0x00 }, /* δ to d */ + { 0x03B5, 0x65, 0x00 }, /* ε to e */ + { 0x03B6, 0x7a, 0x00 }, /* ζ to z */ + { 0x03B7, 0x69, 0x00 }, /* η to i */ + { 0x03B8, 0x74, 0x68 }, /* θ to th */ + { 0x03B9, 0x69, 0x00 }, /* ι to i */ + { 0x03BA, 0x6b, 0x00 }, /* κ to k */ + { 0x03BB, 0x6c, 0x00 }, /* λ to l */ + { 0x03BC, 0x6d, 0x00 }, /* μ to m */ + { 0x03BD, 0x6e, 0x00 }, /* ν to n */ + { 0x03BE, 0x78, 0x00 }, /* ξ to x */ + { 0x03BF, 0x6f, 0x00 }, /* ο to o */ + { 0x03C0, 0x70, 0x00 }, /* π to p */ + { 0x03C1, 0x72, 0x00 }, /* ρ to r */ + { 0x03C3, 0x73, 0x00 }, /* σ to s */ + { 0x03C4, 0x74, 0x00 }, /* τ to t */ + { 0x03C5, 0x79, 0x00 }, /* υ to y */ + { 0x03C6, 0x66, 0x00 }, /* φ to f */ + { 0x03C7, 0x63, 0x68 }, /* χ to ch */ + { 0x03C8, 0x70, 0x73 }, /* ψ to ps */ + { 0x03C9, 0x6f, 0x00 }, /* ω to o */ + { 0x03CA, 0x69, 0x00 }, /* ϊ to i */ + { 0x03CB, 0x79, 0x00 }, /* ϋ to y */ + { 0x03CC, 0x6f, 0x00 }, /* ό to o */ + { 0x03CD, 0x79, 0x00 }, /* ύ to y */ + { 0x03CE, 0x69, 0x00 }, /* ώ to i */ + { 0x0400, 0x45, 0x00 }, /* Ѐ to E */ + { 0x0401, 0x45, 0x00 }, /* Ё to E */ + { 0x0402, 0x44, 0x00 }, /* Ђ to D */ + { 0x0403, 0x47, 0x00 }, /* Ѓ to G */ + { 0x0404, 0x45, 0x00 }, /* Є to E */ + { 0x0405, 0x5a, 0x00 }, /* Ѕ to Z */ + { 0x0406, 0x49, 0x00 }, /* І to I */ + { 0x0407, 0x49, 0x00 }, /* Ї to I */ + { 0x0408, 0x4a, 0x00 }, /* Ј to J */ + { 0x0409, 0x49, 0x00 }, /* Љ to I */ + { 0x040A, 0x4e, 0x00 }, /* Њ to N */ + { 0x040B, 0x44, 0x00 }, /* Ћ to D */ + { 0x040C, 0x4b, 0x00 }, /* Ќ to K */ + { 0x040D, 0x49, 0x00 }, /* Ѝ to I */ + { 0x040E, 0x55, 0x00 }, /* Ў to U */ + { 0x040F, 0x44, 0x00 }, /* Џ to D */ + { 0x0410, 0x41, 0x00 }, /* А to A */ + { 0x0411, 0x42, 0x00 }, /* Б to B */ + { 0x0412, 0x56, 0x00 }, /* В to V */ + { 0x0413, 0x47, 0x00 }, /* Г to G */ + { 0x0414, 0x44, 0x00 }, /* Д to D */ + { 0x0415, 0x45, 0x00 }, /* Е to E */ + { 0x0416, 0x5a, 0x68 }, /* Ж to Zh */ + { 0x0417, 0x5a, 0x00 }, /* З to Z */ + { 0x0418, 0x49, 0x00 }, /* И to I */ + { 0x0419, 0x49, 0x00 }, /* Й to I */ + { 0x041A, 0x4b, 0x00 }, /* К to K */ + { 0x041B, 0x4c, 0x00 }, /* Л to L */ + { 0x041C, 0x4d, 0x00 }, /* М to M */ + { 0x041D, 0x4e, 0x00 }, /* Н to N */ + { 0x041E, 0x4f, 0x00 }, /* О to O */ + { 0x041F, 0x50, 0x00 }, /* П to P */ + { 0x0420, 0x52, 0x00 }, /* Р to R */ + { 0x0421, 0x53, 0x00 }, /* С to S */ + { 0x0422, 0x54, 0x00 }, /* Т to T */ + { 0x0423, 0x55, 0x00 }, /* У to U */ + { 0x0424, 0x46, 0x00 }, /* Ф to F */ + { 0x0425, 0x4b, 0x68 }, /* Х to Kh */ + { 0x0426, 0x54, 0x63 }, /* Ц to Tc */ + { 0x0427, 0x43, 0x68 }, /* Ч to Ch */ + { 0x0428, 0x53, 0x68 }, /* Ш to Sh */ + { 0x0429, 0x53, 0x68 }, /* Щ to Shch */ + { 0x042B, 0x59, 0x00 }, /* Ы to Y */ + { 0x042D, 0x45, 0x00 }, /* Э to E */ + { 0x042E, 0x49, 0x75 }, /* Ю to Iu */ + { 0x042F, 0x49, 0x61 }, /* Я to Ia */ + { 0x0430, 0x61, 0x00 }, /* а to a */ + { 0x0431, 0x62, 0x00 }, /* б to b */ + { 0x0432, 0x76, 0x00 }, /* в to v */ + { 0x0433, 0x67, 0x00 }, /* г to g */ + { 0x0434, 0x64, 0x00 }, /* д to d */ + { 0x0435, 0x65, 0x00 }, /* е to e */ + { 0x0436, 0x7a, 0x68 }, /* ж to zh */ + { 0x0437, 0x7a, 0x00 }, /* з to z */ + { 0x0438, 0x69, 0x00 }, /* и to i */ + { 0x0439, 0x69, 0x00 }, /* й to i */ + { 0x043A, 0x6b, 0x00 }, /* к to k */ + { 0x043B, 0x6c, 0x00 }, /* л to l */ + { 0x043C, 0x6d, 0x00 }, /* м to m */ + { 0x043D, 0x6e, 0x00 }, /* н to n */ + { 0x043E, 0x6f, 0x00 }, /* о to o */ + { 0x043F, 0x70, 0x00 }, /* п to p */ + { 0x0440, 0x72, 0x00 }, /* р to r */ + { 0x0441, 0x73, 0x00 }, /* с to s */ + { 0x0442, 0x74, 0x00 }, /* т to t */ + { 0x0443, 0x75, 0x00 }, /* у to u */ + { 0x0444, 0x66, 0x00 }, /* ф to f */ + { 0x0445, 0x6b, 0x68 }, /* х to kh */ + { 0x0446, 0x74, 0x63 }, /* ц to tc */ + { 0x0447, 0x63, 0x68 }, /* ч to ch */ + { 0x0448, 0x73, 0x68 }, /* ш to sh */ + { 0x0449, 0x73, 0x68 }, /* щ to shch */ + { 0x044B, 0x79, 0x00 }, /* ы to y */ + { 0x044D, 0x65, 0x00 }, /* э to e */ + { 0x044E, 0x69, 0x75 }, /* ю to iu */ + { 0x044F, 0x69, 0x61 }, /* я to ia */ + { 0x0450, 0x65, 0x00 }, /* ѐ to e */ + { 0x0451, 0x65, 0x00 }, /* ё to e */ + { 0x0452, 0x64, 0x00 }, /* ђ to d */ + { 0x0453, 0x67, 0x00 }, /* ѓ to g */ + { 0x0454, 0x65, 0x00 }, /* є to e */ + { 0x0455, 0x7a, 0x00 }, /* ѕ to z */ + { 0x0456, 0x69, 0x00 }, /* і to i */ + { 0x0457, 0x69, 0x00 }, /* ї to i */ + { 0x0458, 0x6a, 0x00 }, /* ј to j */ + { 0x0459, 0x69, 0x00 }, /* љ to i */ + { 0x045A, 0x6e, 0x00 }, /* њ to n */ + { 0x045B, 0x64, 0x00 }, /* ћ to d */ + { 0x045C, 0x6b, 0x00 }, /* ќ to k */ + { 0x045D, 0x69, 0x00 }, /* ѝ to i */ + { 0x045E, 0x75, 0x00 }, /* ў to u */ + { 0x045F, 0x64, 0x00 }, /* џ to d */ + { 0x1E02, 0x42, 0x00 }, /* Ḃ to B */ + { 0x1E03, 0x62, 0x00 }, /* ḃ to b */ + { 0x1E0A, 0x44, 0x00 }, /* Ḋ to D */ + { 0x1E0B, 0x64, 0x00 }, /* ḋ to d */ + { 0x1E1E, 0x46, 0x00 }, /* Ḟ to F */ + { 0x1E1F, 0x66, 0x00 }, /* ḟ to f */ + { 0x1E40, 0x4D, 0x00 }, /* Ṁ to M */ + { 0x1E41, 0x6D, 0x00 }, /* ṁ to m */ + { 0x1E56, 0x50, 0x00 }, /* Ṗ to P */ + { 0x1E57, 0x70, 0x00 }, /* ṗ to p */ + { 0x1E60, 0x53, 0x00 }, /* Ṡ to S */ + { 0x1E61, 0x73, 0x00 }, /* ṡ to s */ + { 0x1E6A, 0x54, 0x00 }, /* Ṫ to T */ + { 0x1E6B, 0x74, 0x00 }, /* ṫ to t */ + { 0x1E80, 0x57, 0x00 }, /* Ẁ to W */ + { 0x1E81, 0x77, 0x00 }, /* ẁ to w */ + { 0x1E82, 0x57, 0x00 }, /* Ẃ to W */ + { 0x1E83, 0x77, 0x00 }, /* ẃ to w */ + { 0x1E84, 0x57, 0x00 }, /* Ẅ to W */ + { 0x1E85, 0x77, 0x00 }, /* ẅ to w */ + { 0x1EF2, 0x59, 0x00 }, /* Ỳ to Y */ + { 0x1EF3, 0x79, 0x00 }, /* ỳ to y */ + { 0xFB00, 0x66, 0x66 }, /* ff to ff */ + { 0xFB01, 0x66, 0x69 }, /* fi to fi */ + { 0xFB02, 0x66, 0x6C }, /* fl to fl */ + { 0xFB05, 0x73, 0x74 }, /* ſt to st */ + { 0xFB06, 0x73, 0x74 }, /* st to st */ +}; + +/* +** Convert the input string from UTF-8 into pure ASCII by converting +** all non-ASCII characters to some combination of characters in the +** ASCII subset. +** +** The returned string might contain more characters than the input. +** +** Space to hold the returned string comes from sqlite3_malloc() and +** should be freed by the caller. +*/ +static unsigned char *transliterate(const unsigned char *zIn, int nIn){ + unsigned char *zOut = sqlite3_malloc( nIn*4 + 1 ); + int i, c, sz, nOut; + if( zOut==0 ) return 0; + i = nOut = 0; + while( i<nIn ){ + c = utf8Read(zIn, nIn, &sz); + zIn += sz; + nIn -= sz; + if( c<=127 ){ + zOut[nOut++] = c; + }else{ + int xTop, xBtm, x; + xTop = sizeof(translit)/sizeof(translit[0]) - 1; + xBtm = 0; + while( xTop>=xBtm ){ + x = (xTop + xBtm)/2; + if( translit[x].cFrom==c ){ + zOut[nOut++] = translit[x].cTo0; + if( translit[x].cTo1 ){ + zOut[nOut++] = translit[x].cTo1; + /* Add an extra "ch" after the "sh" for Щ and щ */ + if( c==0x0429 || c== 0x0449 ){ + zOut[nOut++] = 'c'; + zOut[nOut++] = 'h'; + } + } + c = 0; + break; + }else if( translit[x].cFrom>c ){ + xTop = x-1; + }else{ + xBtm = x+1; + } + } + if( c ) zOut[nOut++] = '?'; + } + } + zOut[nOut] = 0; + return zOut; +} + +/* +** spellfix1_translit(X) +** +** Convert a string that contains non-ASCII Roman characters into +** pure ASCII. +*/ +static void transliterateSqlFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zIn = sqlite3_value_text(argv[0]); + int nIn = sqlite3_value_bytes(argv[0]); + unsigned char *zOut = transliterate(zIn, nIn); + if( zOut==0 ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_result_text(context, (char*)zOut, -1, sqlite3_free); + } +} + +/* +** spellfix1_scriptcode(X) +** +** Try to determine the dominant script used by the word X and return +** its ISO 15924 numeric code. +** +** The current implementation only understands the following scripts: +** +** 215 (Latin) +** 220 (Cyrillic) +** 200 (Greek) +** +** This routine will return 998 if the input X contains characters from +** two or more of the above scripts or 999 if X contains no characters +** from any of the above scripts. +*/ +static void scriptCodeSqlFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zIn = sqlite3_value_text(argv[0]); + int nIn = sqlite3_value_bytes(argv[0]); + int c, sz; + int scriptMask = 0; + int res; +# define SCRIPT_LATIN 0x0001 +# define SCRIPT_CYRILLIC 0x0002 +# define SCRIPT_GREEK 0x0004 + + while( nIn>0 ){ + c = utf8Read(zIn, nIn, &sz); + zIn += sz; + nIn -= sz; + if( c<0x02af ){ + scriptMask |= SCRIPT_LATIN; + }else if( c>=0x0400 && c<=0x04ff ){ + scriptMask |= SCRIPT_CYRILLIC; + }else if( c>=0x0386 && c<=0x03ce ){ + scriptMask |= SCRIPT_GREEK; + } + } + switch( scriptMask ){ + case 0: res = 999; break; + case SCRIPT_LATIN: res = 215; break; + case SCRIPT_CYRILLIC: res = 220; break; + case SCRIPT_GREEK: res = 200; break; + default: res = 998; break; + } + sqlite3_result_int(context, res); +} + +/***************************************************************************** +** Fuzzy-search virtual table +*****************************************************************************/ + +typedef struct spellfix1_vtab spellfix1_vtab; +typedef struct spellfix1_cursor spellfix1_cursor; + +/* Fuzzy-search virtual table object */ +struct spellfix1_vtab { + sqlite3_vtab base; /* Base class - must be first */ + sqlite3 *db; /* Database connection */ + char *zDbName; /* Name of database holding this table */ + char *zTableName; /* Name of the virtual table */ +}; + +/* Fuzzy-search cursor object */ +struct spellfix1_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + spellfix1_vtab *pVTab; /* The table to which this cursor belongs */ + int nRow; /* Number of rows of content */ + int nAlloc; /* Number of allocated rows */ + int iRow; /* Current row of content */ + int iLang; /* Value of the lang= constraint */ + int iTop; /* Value of the top= constraint */ + int iScope; /* Value of the scope= constraint */ + int nSearch; /* Number of vocabulary items checked */ + struct spellfix1_row { /* For each row of content */ + sqlite3_int64 iRowid; /* Rowid for this row */ + char *zWord; /* Text for this row */ + int iRank; /* Rank for this row */ + int iDistance; /* Distance from pattern for this row */ + int iScore; /* Score for sorting */ + } *a; +}; + +/* +** Construct one or more SQL statements from the format string given +** and then evaluate those statements. The success code is written +** into *pRc. +** +** If *pRc is initially non-zero then this routine is a no-op. +*/ +static void spellfix1DbExec( + int *pRc, /* Success code */ + sqlite3 *db, /* Database in which to run SQL */ + const char *zFormat, /* Format string for SQL */ + ... /* Arguments to the format string */ +){ + va_list ap; + char *zSql; + if( *pRc ) return; + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + if( zSql==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + *pRc = sqlite3_exec(db, zSql, 0, 0, 0); + sqlite3_free(zSql); + } +} + +/* +** xDisconnect/xDestroy method for the fuzzy-search module. +*/ +static int spellfix1Uninit(int isDestroy, sqlite3_vtab *pVTab){ + spellfix1_vtab *p = (spellfix1_vtab*)pVTab; + int rc = SQLITE_OK; + if( isDestroy ){ + sqlite3 *db = p->db; + spellfix1DbExec(&rc, db, "DROP TABLE IF EXISTS \"%w\".\"%w_vocab\"", + p->zDbName, p->zTableName); + } + if( rc==SQLITE_OK ){ + sqlite3_free(p->zTableName); + sqlite3_free(p); + } + return rc; +} +static int spellfix1Disconnect(sqlite3_vtab *pVTab){ + return spellfix1Uninit(0, pVTab); +} +static int spellfix1Destroy(sqlite3_vtab *pVTab){ + return spellfix1Uninit(1, pVTab); +} + +/* +** xConnect/xCreate method for the spellfix1 module. Arguments are: +** +** argv[0] -> module name ("spellfix1") +** argv[1] -> database name +** argv[2] -> table name +** argv[3].. -> optional arguments (currently ignored) +*/ +static int spellfix1Init( + int isCreate, + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVTab, + char **pzErr +){ + spellfix1_vtab *pNew = 0; + const char *zModule = argv[0]; + const char *zDbName = argv[1]; + const char *zTableName = argv[2]; + int nDbName; + int rc = SQLITE_OK; + + if( argc<3 ){ + *pzErr = sqlite3_mprintf( + "%s: wrong number of CREATE VIRTUAL TABLE arguments", argv[0] + ); + rc = SQLITE_ERROR; + }else{ + nDbName = strlen(zDbName); + pNew = sqlite3_malloc( sizeof(*pNew) + nDbName + 1); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pNew, 0, sizeof(*pNew)); + pNew->zDbName = (char*)&pNew[1]; + memcpy(pNew->zDbName, zDbName, nDbName+1); + pNew->zTableName = sqlite3_mprintf("%s", zTableName); + pNew->db = db; + if( pNew->zTableName==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(word,rank,distance,langid," + "score,top HIDDEN,scope HIDDEN,srchcnt HIDDEN," + "soundslike HIDDEN)" + ); + } + if( rc==SQLITE_OK && isCreate ){ + sqlite3_uint64 r; + spellfix1DbExec(&rc, db, + "CREATE TABLE IF NOT EXISTS \"%w\".\"%w_vocab\"(\n" + " id INTEGER PRIMARY KEY,\n" + " rank INT,\n" + " langid INT,\n" + " word TEXT,\n" + " k1 TEXT,\n" + " k2 TEXT\n" + ");\n", + zDbName, zTableName + ); + sqlite3_randomness(sizeof(r), &r); + spellfix1DbExec(&rc, db, + "CREATE INDEX IF NOT EXISTS \"%w\".\"%w_index_%llx\" " + "ON \"%w_vocab\"(langid,k2);", + zDbName, zModule, r, zTableName + ); + } + } + } + + *ppVTab = (sqlite3_vtab *)pNew; + return rc; +} + +/* +** The xConnect and xCreate methods +*/ +static int spellfix1Connect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVTab, + char **pzErr +){ + return spellfix1Init(0, db, pAux, argc, argv, ppVTab, pzErr); +} +static int spellfix1Create( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVTab, + char **pzErr +){ + return spellfix1Init(1, db, pAux, argc, argv, ppVTab, pzErr); +} + +/* +** Reset a cursor so that it contains zero rows of content but holds +** space for N rows. +*/ +static void spellfix1ResetCursor(spellfix1_cursor *pCur, int N){ + int i; + for(i=0; i<pCur->nRow; i++){ + sqlite3_free(pCur->a[i].zWord); + } + pCur->a = sqlite3_realloc(pCur->a, sizeof(pCur->a[0])*N); + pCur->nAlloc = N; + pCur->nRow = 0; + pCur->iRow = 0; + pCur->nSearch = 0; +} + +/* +** Close a fuzzy-search cursor. +*/ +static int spellfix1Close(sqlite3_vtab_cursor *cur){ + spellfix1_cursor *pCur = (spellfix1_cursor *)cur; + spellfix1ResetCursor(pCur, 0); + sqlite3_free(pCur); + return SQLITE_OK; +} + +/* +** Search for terms of these forms: +** +** (A) word MATCH $str +** (B) langid == $langid +** (C) top = $top +** (D) scope = $scope +** +** The plan number is a bit mask formed with these bits: +** +** 0x01 (A) is found +** 0x02 (B) is found +** 0x04 (C) is found +** 0x08 (D) is found +** +** filter.argv[*] values contains $str, $langid, $top, and $scope, +** if specified and in that order. +*/ +static int spellfix1BestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + int iPlan = 0; + int iLangTerm = -1; + int iTopTerm = -1; + int iScopeTerm = -1; + int i; + const struct sqlite3_index_constraint *pConstraint; + pConstraint = pIdxInfo->aConstraint; + for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + + /* Terms of the form: word MATCH $str */ + if( (iPlan & 1)==0 + && pConstraint->iColumn==0 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH + ){ + iPlan |= 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + } + + /* Terms of the form: langid = $langid */ + if( (iPlan & 2)==0 + && pConstraint->iColumn==3 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= 2; + iLangTerm = i; + } + + /* Terms of the form: top = $top */ + if( (iPlan & 4)==0 + && pConstraint->iColumn==5 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= 4; + iTopTerm = i; + } + + /* Terms of the form: scope = $scope */ + if( (iPlan & 8)==0 + && pConstraint->iColumn==6 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= 8; + iScopeTerm = i; + } + } + if( iPlan&1 ){ + int idx = 2; + pIdxInfo->idxNum = iPlan; + if( pIdxInfo->nOrderBy==1 + && pIdxInfo->aOrderBy[0].iColumn==4 + && pIdxInfo->aOrderBy[0].desc==0 + ){ + pIdxInfo->orderByConsumed = 1; /* Default order by iScore */ + } + if( iPlan&2 ){ + pIdxInfo->aConstraintUsage[iLangTerm].argvIndex = idx++; + pIdxInfo->aConstraintUsage[iLangTerm].omit = 1; + } + if( iPlan&4 ){ + pIdxInfo->aConstraintUsage[iTopTerm].argvIndex = idx++; + pIdxInfo->aConstraintUsage[iTopTerm].omit = 1; + } + if( iPlan&8 ){ + pIdxInfo->aConstraintUsage[iScopeTerm].argvIndex = idx++; + pIdxInfo->aConstraintUsage[iScopeTerm].omit = 1; + } + pIdxInfo->estimatedCost = (double)10000; + }else{ + pIdxInfo->idxNum = 0; + pIdxInfo->estimatedCost = (double)10000000; + } + return SQLITE_OK; +} + +/* +** Open a new fuzzy-search cursor. +*/ +static int spellfix1Open(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + spellfix1_vtab *p = (spellfix1_vtab*)pVTab; + spellfix1_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->pVTab = p; + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Adjust a distance measurement by the words rank in order to show +** preference to common words. +*/ +static int spellfix1Score(int iDistance, int iRank){ + int iLog2; + for(iLog2=0; iRank>0; iLog2++, iRank>>=1){} + return iDistance + 32 - iLog2; +} + +/* +** Compare two spellfix1_row objects for sorting purposes in qsort() such +** that they sort in order of increasing distance. +*/ +static int spellfix1RowCompare(const void *A, const void *B){ + const struct spellfix1_row *a = (const struct spellfix1_row*)A; + const struct spellfix1_row *b = (const struct spellfix1_row*)B; + return a->iScore - b->iScore; +} + +/* +** This version of the xFilter method work if the MATCH term is present +** and we are doing a scan. +*/ +static int spellfix1FilterForMatch( + spellfix1_cursor *pCur, + int idxNum, + int argc, + sqlite3_value **argv +){ + const unsigned char *zPatternIn; + char *zPattern; + int nPattern; + char *zClass; + int nClass; + int iLimit = 20; + int iScope = 4; + int iLang = 0; + char *zSql; + int rc; + sqlite3_stmt *pStmt; + int idx = 1; + spellfix1_vtab *p = pCur->pVTab; + + if( idxNum&2 ){ + iLang = sqlite3_value_int(argv[idx++]); + } + if( idxNum&4 ){ + iLimit = sqlite3_value_int(argv[idx++]); + if( iLimit<1 ) iLimit = 1; + } + if( idxNum&8 ){ + iScope = sqlite3_value_int(argv[idx++]); + if( iScope<1 ) iScope = 1; + } + spellfix1ResetCursor(pCur, iLimit); + zPatternIn = sqlite3_value_text(argv[0]); + if( zPatternIn==0 ) return SQLITE_OK; + zPattern = (char*)transliterate(zPatternIn, sqlite3_value_bytes(argv[0])); + if( zPattern==0 ) return SQLITE_NOMEM; + nPattern = strlen(zPattern); + if( zPattern[nPattern-1]=='*' ) nPattern--; + if( nPattern<iScope ) iScope = nPattern; + zClass = (char*)characterClassString((unsigned char*)zPattern, + strlen(zPattern)); + nClass = strlen(zClass); + if( nClass>iScope ){ + zClass[iScope] = 0; + nClass = iScope; + } + zSql = sqlite3_mprintf( + "SELECT id, word, rank, k1" + " FROM \"%w\".\"%w_vocab\"" + " WHERE langid=%d AND k2 GLOB '%q*'", + p->zDbName, p->zTableName, iLang, zClass + ); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc==SQLITE_OK ){ + const char *zK1; + int iDist; + int iRank; + int iScore; + int iWorst = 999999999; + int idx; + int idxWorst; + int i; + + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + zK1 = (const char*)sqlite3_column_text(pStmt, 3); + if( zK1==0 ) continue; + pCur->nSearch++; + iRank = sqlite3_column_int(pStmt, 2); + iDist = editdist(zPattern, zK1); + iScore = spellfix1Score(iDist,iRank); + if( pCur->nRow<pCur->nAlloc ){ + idx = pCur->nRow; + }else if( iScore<iWorst ){ + idx = idxWorst; + sqlite3_free(pCur->a[idx].zWord); + }else{ + continue; + } + pCur->a[idx].zWord = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1)); + pCur->a[idx].iRowid = sqlite3_column_int64(pStmt, 0); + pCur->a[idx].iRank = iRank; + pCur->a[idx].iDistance = iDist; + pCur->a[idx].iScore = iScore; + if( pCur->nRow<pCur->nAlloc ) pCur->nRow++; + if( pCur->nRow==pCur->nAlloc ){ + iWorst = pCur->a[0].iScore; + idxWorst = 0; + for(i=1; i<pCur->nRow; i++){ + iScore = pCur->a[i].iScore; + if( iWorst<iScore ){ + iWorst = iScore; + idxWorst = i; + } + } + } + } + } + qsort(pCur->a, pCur->nRow, sizeof(pCur->a[0]), spellfix1RowCompare); + pCur->iTop = iLimit; + pCur->iScope = iScope; + sqlite3_finalize(pStmt); + sqlite3_free(zPattern); + sqlite3_free(zClass); + return SQLITE_OK; +} + +/* +** This version of xFilter handles a full-table scan case +*/ +static int spellfix1FilterForFullScan( + spellfix1_cursor *pCur, + int idxNum, + int argc, + sqlite3_value **argv +){ + spellfix1ResetCursor(pCur, 0); + return SQLITE_OK; +} + + +/* +** Called to "rewind" a cursor back to the beginning so that +** it starts its output over again. Always called at least once +** prior to any spellfix1Column, spellfix1Rowid, or spellfix1Eof call. +*/ +static int spellfix1Filter( + sqlite3_vtab_cursor *cur, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + spellfix1_cursor *pCur = (spellfix1_cursor *)cur; + int rc; + if( idxNum & 1 ){ + rc = spellfix1FilterForMatch(pCur, idxNum, argc, argv); + }else{ + rc = spellfix1FilterForFullScan(pCur, idxNum, argc, argv); + } + return rc; +} + + +/* +** Advance a cursor to its next row of output +*/ +static int spellfix1Next(sqlite3_vtab_cursor *cur){ + spellfix1_cursor *pCur = (spellfix1_cursor *)cur; + if( pCur->iRow < pCur->nRow ) pCur->iRow++; + return SQLITE_OK; +} + +/* +** Return TRUE if we are at the end-of-file +*/ +static int spellfix1Eof(sqlite3_vtab_cursor *cur){ + spellfix1_cursor *pCur = (spellfix1_cursor *)cur; + return pCur->iRow>=pCur->nRow; +} + +/* +** Return columns from the current row. +*/ +static int spellfix1Column(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ + spellfix1_cursor *pCur = (spellfix1_cursor*)cur; + switch( i ){ + case 0: { + sqlite3_result_text(ctx, pCur->a[pCur->iRow].zWord, -1, SQLITE_STATIC); + break; + } + case 1: { + sqlite3_result_int(ctx, pCur->a[pCur->iRow].iRank); + break; + } + case 2: { + sqlite3_result_int(ctx, pCur->a[pCur->iRow].iDistance); + break; + } + case 3: { + sqlite3_result_int(ctx, pCur->iLang); + break; + } + case 4: { + sqlite3_result_int(ctx, pCur->a[pCur->iRow].iScore); + break; + } + case 5: { + sqlite3_result_int(ctx, pCur->iTop); + break; + } + case 6: { + sqlite3_result_int(ctx, pCur->iScope); + break; + } + case 7: { + sqlite3_result_int(ctx, pCur->nSearch); + break; + } + default: { + sqlite3_result_null(ctx); + break; + } + } + return SQLITE_OK; +} + +/* +** The rowid. +*/ +static int spellfix1Rowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + spellfix1_cursor *pCur = (spellfix1_cursor*)cur; + *pRowid = pCur->a[pCur->iRow].iRowid; + return SQLITE_OK; +} + +/* +** The xUpdate() method. +*/ +static int spellfix1Update( + sqlite3_vtab *pVTab, + int argc, + sqlite3_value **argv, + sqlite_int64 *pRowid +){ + int rc = SQLITE_OK; + sqlite3_int64 rowid, newRowid; + spellfix1_vtab *p = (spellfix1_vtab*)pVTab; + sqlite3 *db = p->db; + + if( argc==1 ){ + /* A delete operation on the rowid given by argv[0] */ + rowid = *pRowid = sqlite3_value_int64(argv[0]); + spellfix1DbExec(&rc, db, "DELETE FROM \"%w\".\"%w_vocab\" " + " WHERE id=%lld", + p->zDbName, p->zTableName, rowid); + }else{ + const unsigned char *zWord = sqlite3_value_text(argv[2]); + int nWord = sqlite3_value_bytes(argv[2]); + int iLang = sqlite3_value_int(argv[5]); + int iRank = sqlite3_value_int(argv[3]); + const unsigned char *zSoundslike = sqlite3_value_text(argv[10]); + int nSoundslike = sqlite3_value_bytes(argv[10]); + char *zK1, *zK2; + int i; + char c; + + if( zWord==0 ){ + pVTab->zErrMsg = sqlite3_mprintf("%w.word may not be NULL", + p->zTableName); + return SQLITE_CONSTRAINT; + } + if( iRank<1 ) iRank = 1; + if( zSoundslike ){ + zK1 = (char*)transliterate(zSoundslike, nSoundslike); + }else{ + zK1 = (char*)transliterate(zWord, nWord); + } + if( zK1==0 ) return SQLITE_NOMEM; + for(i=0; (c = zK1[i])!=0; i++){ + if( c>='A' && c<='Z' ) zK1[i] += 'a' - 'A'; + } + zK2 = (char*)characterClassString((const unsigned char*)zK1, i); + if( zK2==0 ){ + sqlite3_free(zK1); + return SQLITE_NOMEM; + } + if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + spellfix1DbExec(&rc, db, + "INSERT INTO \"%w\".\"%w_vocab\"(rank,langid,word,k1,k2) " + "VALUES(%d,%d,%Q,%Q,%Q)", + p->zDbName, p->zTableName, + iRank, iLang, zWord, zK1, zK2 + ); + *pRowid = sqlite3_last_insert_rowid(db); + }else{ + rowid = sqlite3_value_int64(argv[0]); + newRowid = *pRowid = sqlite3_value_int64(argv[1]); + spellfix1DbExec(&rc, db, + "UPDATE \"%w\".\"%w_vocab\" SET id=%lld, rank=%d, lang=%d," + " word=%Q, rank=%d, k1=%Q, k2=%Q WHERE id=%lld", + p->zDbName, p->zTableName, newRowid, iRank, iLang, + zWord, zK1, zK2, rowid + ); + } + sqlite3_free(zK1); + sqlite3_free(zK2); + } + return rc; +} + +/* +** Rename the spellfix1 table. +*/ +static int spellfix1Rename(sqlite3_vtab *pVTab, const char *zNew){ + spellfix1_vtab *p = (spellfix1_vtab*)pVTab; + sqlite3 *db = p->db; + int rc = SQLITE_OK; + char *zNewName = sqlite3_mprintf("%s", zNew); + if( zNewName==0 ){ + return SQLITE_NOMEM; + } + spellfix1DbExec(&rc, db, + "ALTER TABLE \"%w\".\"%w_vocab\" RENAME TO \"%w_vocab\"", + p->zDbName, p->zTableName, zNewName + ); + if( rc==SQLITE_OK ){ + sqlite3_free(p->zTableName); + p->zTableName = zNewName; + } + return rc; +} + + +/* +** A virtual table module that provides fuzzy search. +*/ +static sqlite3_module spellfix1Module = { + 0, /* iVersion */ + spellfix1Create, /* xCreate - handle CREATE VIRTUAL TABLE */ + spellfix1Connect, /* xConnect - reconnected to an existing table */ + spellfix1BestIndex, /* xBestIndex - figure out how to do a query */ + spellfix1Disconnect, /* xDisconnect - close a connection */ + spellfix1Destroy, /* xDestroy - handle DROP TABLE */ + spellfix1Open, /* xOpen - open a cursor */ + spellfix1Close, /* xClose - close a cursor */ + spellfix1Filter, /* xFilter - configure scan constraints */ + spellfix1Next, /* xNext - advance a cursor */ + spellfix1Eof, /* xEof - check for end of scan */ + spellfix1Column, /* xColumn - read data */ + spellfix1Rowid, /* xRowid - read data */ + spellfix1Update, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + spellfix1Rename, /* xRename */ +}; + +/* +** Register the various functions and the virtual table. +*/ +static int spellfix1Register(sqlite3 *db){ + int nErr = 0; + int i; + nErr += sqlite3_create_function(db, "spellfix1_translit", 1, SQLITE_UTF8, 0, + transliterateSqlFunc, 0, 0); + nErr += sqlite3_create_function(db, "spellfix1_editdist", 2, SQLITE_UTF8, 0, + editdistSqlFunc, 0, 0); + nErr += sqlite3_create_function(db, "spellfix1_charclass", 1, SQLITE_UTF8, 0, + characterClassSqlFunc, 0, 0); + nErr += sqlite3_create_function(db, "spellfix1_scriptcode", 1, SQLITE_UTF8, 0, + scriptCodeSqlFunc, 0, 0); + nErr += sqlite3_create_module(db, "spellfix1", &spellfix1Module, 0); + + /* Verify sanity of the translit[] table */ + for(i=0; i<sizeof(translit)/sizeof(translit[0])-1; i++){ + assert( translit[i].cFrom<translit[i+1].cFrom ); + } + + return nErr ? SQLITE_ERROR : SQLITE_OK; +} + +#if SQLITE_CORE || defined(SQLITE_TEST) +/* +** Register the spellfix1 virtual table and its associated functions. +*/ +int sqlite3Spellfix1Register(sqlite3 *db){ + return spellfix1Register(db); +} +#endif + + +#if !SQLITE_CORE +/* +** Extension load function. +*/ +int sqlite3_extension_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + return spellfix1Register(db); +} +#endif /* !SQLITE_CORE */ diff --git a/lib/libsqlite3/src/test_stat.c b/lib/libsqlite3/src/test_stat.c index a0893b39747..d4c902b5ea1 100644 --- a/lib/libsqlite3/src/test_stat.c +++ b/lib/libsqlite3/src/test_stat.c @@ -324,12 +324,13 @@ static int statDecodePage(Btree *pBt, StatPage *p){ u64 dummy; iOff += sqlite3GetVarint(&aData[iOff], &dummy); } - if( nPayload>p->nMxPayload ) p->nMxPayload = nPayload; + if( nPayload>(u32)p->nMxPayload ) p->nMxPayload = nPayload; getLocalPayload(nUsable, p->flags, nPayload, &nLocal); pCell->nLocal = nLocal; - assert( nPayload>=nLocal ); + assert( nLocal>=0 ); + assert( nPayload>=(u32)nLocal ); assert( nLocal<=(nUsable-35) ); - if( nPayload>nLocal ){ + if( nPayload>(u32)nLocal ){ int j; int nOvfl = ((nPayload - nLocal) + nUsable-4 - 1) / (nUsable - 4); pCell->nLastOvfl = (nPayload-nLocal) - (nOvfl-1) * (nUsable-4); @@ -378,7 +379,7 @@ static void statSizeAndOffset(StatCursor *pCsr){ x[0] = pCsr->iPageno; if( sqlite3OsFileControl(fd, 230440, &x)==SQLITE_OK ){ pCsr->iOffset = x[0]; - pCsr->szPage = x[1]; + pCsr->szPage = (int)x[1]; } } @@ -400,7 +401,7 @@ static int statNext(sqlite3_vtab_cursor *pCursor){ rc = sqlite3_step(pCsr->pStmt); if( rc==SQLITE_ROW ){ int nPage; - u32 iRoot = sqlite3_column_int64(pCsr->pStmt, 1); + u32 iRoot = (u32)sqlite3_column_int64(pCsr->pStmt, 1); sqlite3PagerPagecount(pPager, &nPage); if( nPage==0 ){ pCsr->isEof = 1; diff --git a/lib/libsqlite3/src/test_vfs.c b/lib/libsqlite3/src/test_vfs.c index bab67798cc0..d1c34a38e4c 100644 --- a/lib/libsqlite3/src/test_vfs.c +++ b/lib/libsqlite3/src/test_vfs.c @@ -784,7 +784,7 @@ static int tvfsShmOpen(sqlite3_file *pFile){ if( 0==strcmp(pFd->zFilename, pBuffer->zFile) ) break; } if( !pBuffer ){ - int nByte = sizeof(TestvfsBuffer) + strlen(pFd->zFilename) + 1; + int nByte = sizeof(TestvfsBuffer) + (int)strlen(pFd->zFilename) + 1; pBuffer = (TestvfsBuffer *)ckalloc(nByte); memset(pBuffer, 0, nByte); pBuffer->zFile = (char *)&pBuffer[1]; @@ -866,13 +866,13 @@ static int tvfsShmLock( if( p->pScript && p->mask&TESTVFS_SHMLOCK_MASK ){ sqlite3_snprintf(sizeof(zLock), zLock, "%d %d", ofst, n); - nLock = strlen(zLock); + nLock = (int)strlen(zLock); if( flags & SQLITE_SHM_LOCK ){ strcpy(&zLock[nLock], " lock"); }else{ strcpy(&zLock[nLock], " unlock"); } - nLock += strlen(&zLock[nLock]); + nLock += (int)strlen(&zLock[nLock]); if( flags & SQLITE_SHM_SHARED ){ strcpy(&zLock[nLock], " shared"); }else{ @@ -1396,7 +1396,7 @@ static int testvfs_cmd( } zVfs = Tcl_GetString(objv[1]); - nByte = sizeof(Testvfs) + strlen(zVfs)+1; + nByte = sizeof(Testvfs) + (int)strlen(zVfs)+1; p = (Testvfs *)ckalloc(nByte); memset(p, 0, nByte); p->iDevchar = -1; diff --git a/lib/libsqlite3/src/test_wholenumber.c b/lib/libsqlite3/src/test_wholenumber.c index 150dc95ac49..7c42d01691b 100644 --- a/lib/libsqlite3/src/test_wholenumber.c +++ b/lib/libsqlite3/src/test_wholenumber.c @@ -33,8 +33,8 @@ typedef struct wholenumber_cursor wholenumber_cursor; struct wholenumber_cursor { sqlite3_vtab_cursor base; /* Base class - must be first */ - unsigned iValue; /* Current value */ - unsigned mxValue; /* Maximum value */ + sqlite3_int64 iValue; /* Current value */ + sqlite3_int64 mxValue; /* Maximum value */ }; /* Methods for the wholenumber module */ diff --git a/lib/libsqlite3/src/vdbe.c b/lib/libsqlite3/src/vdbe.c index 749830bc0da..fa5180c9a42 100644 --- a/lib/libsqlite3/src/vdbe.c +++ b/lib/libsqlite3/src/vdbe.c @@ -2127,6 +2127,11 @@ case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */ ** then the cache of the cursor is reset prior to extracting the column. ** The first OP_Column against a pseudo-table after the value of the content ** register has changed should have this bit set. +** +** If the OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG bits are set on P5 when +** the result is guaranteed to only be used as the argument of a length() +** or typeof() function, respectively. The loading of large blobs can be +** skipped for length() and all content loading can be skipped for typeof(). */ case OP_Column: { u32 payloadSize; /* Number of bytes in the record */ @@ -2267,7 +2272,7 @@ case OP_Column: { pC->aRow = 0; } } - /* The following assert is true in all cases accept when + /* The following assert is true in all cases except when ** the database file has been corrupted externally. ** assert( zRec!=0 || avail>=payloadSize || avail>=9 ); */ szHdr = getVarint32((u8*)zData, offset); @@ -2342,11 +2347,11 @@ case OP_Column: { break; } }else{ - /* If i is less that nField, then there are less fields in this + /* If i is less that nField, then there are fewer fields in this ** record than SetNumColumns indicated there are columns in the ** table. Set the offset for any extra columns not present in - ** the record to 0. This tells code below to store a NULL - ** instead of deserializing a value from the record. + ** the record to 0. This tells code below to store the default value + ** for the column instead of deserializing a value from the record. */ aOffset[i] = 0; } @@ -2376,17 +2381,32 @@ case OP_Column: { if( aOffset[p2] ){ assert( rc==SQLITE_OK ); if( zRec ){ + /* This is the common case where the whole row fits on a single page */ VdbeMemRelease(pDest); sqlite3VdbeSerialGet((u8 *)&zRec[aOffset[p2]], aType[p2], pDest); }else{ - len = sqlite3VdbeSerialTypeLen(aType[p2]); - sqlite3VdbeMemMove(&sMem, pDest); - rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len, pC->isIndex, &sMem); - if( rc!=SQLITE_OK ){ - goto op_column_out; + /* This branch happens only when the row overflows onto multiple pages */ + t = aType[p2]; + if( (pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0 + && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0) + ){ + /* Content is irrelevant for the typeof() function and for + ** the length(X) function if X is a blob. So we might as well use + ** bogus content rather than reading content from disk. NULL works + ** for text and blob and whatever is in the payloadSize64 variable + ** will work for everything else. */ + zData = t<12 ? (char*)&payloadSize64 : 0; + }else{ + len = sqlite3VdbeSerialTypeLen(t); + sqlite3VdbeMemMove(&sMem, pDest); + rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len, pC->isIndex, + &sMem); + if( rc!=SQLITE_OK ){ + goto op_column_out; + } + zData = sMem.z; } - zData = sMem.z; - sqlite3VdbeSerialGet((u8*)zData, aType[p2], pDest); + sqlite3VdbeSerialGet((u8*)zData, t, pDest); } pDest->enc = encoding; }else{ @@ -2714,8 +2734,10 @@ case OP_Savepoint: { rc = p->rc; }else{ iSavepoint = db->nSavepoint - iSavepoint - 1; - for(ii=0; ii<db->nDb; ii++){ - sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, SQLITE_ABORT); + if( p1==SAVEPOINT_ROLLBACK ){ + for(ii=0; ii<db->nDb; ii++){ + sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, SQLITE_ABORT); + } } for(ii=0; ii<db->nDb; ii++){ rc = sqlite3BtreeSavepoint(db->aDb[ii].pBt, p1, iSavepoint); diff --git a/lib/libsqlite3/src/vdbeaux.c b/lib/libsqlite3/src/vdbeaux.c index ef339d1e133..caa2bf6700d 100644 --- a/lib/libsqlite3/src/vdbeaux.c +++ b/lib/libsqlite3/src/vdbeaux.c @@ -1239,7 +1239,7 @@ int sqlite3VdbeList( for(j=0; j<nSub; j++){ if( apSub[j]==pOp->p4.pProgram ) break; } - if( j==nSub && SQLITE_OK==sqlite3VdbeMemGrow(pSub, nByte, 1) ){ + if( j==nSub && SQLITE_OK==sqlite3VdbeMemGrow(pSub, nByte, nSub!=0) ){ apSub = (SubProgram **)pSub->z; apSub[nSub++] = pOp->p4.pProgram; pSub->flags |= MEM_Blob; diff --git a/lib/libsqlite3/src/vdbemem.c b/lib/libsqlite3/src/vdbemem.c index 088d3d64a48..fd964de2e91 100644 --- a/lib/libsqlite3/src/vdbemem.c +++ b/lib/libsqlite3/src/vdbemem.c @@ -59,10 +59,10 @@ int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){ ** Make sure pMem->z points to a writable allocation of at least ** n bytes. ** -** If the memory cell currently contains string or blob data -** and the third argument passed to this function is true, the -** current content of the cell is preserved. Otherwise, it may -** be discarded. +** If the third argument passed to this function is true, then memory +** cell pMem must contain a string or blob. In this case the content is +** preserved. Otherwise, if the third parameter to this function is false, +** any current string or blob value may be discarded. ** ** This function sets the MEM_Dyn flag and clears any xDel callback. ** It also clears MEM_Ephem and MEM_Static. If the preserve flag is @@ -77,6 +77,10 @@ int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve){ ); assert( (pMem->flags&MEM_RowSet)==0 ); + /* If the preserve flag is set to true, then the memory cell must already + ** contain a valid string or blob value. */ + assert( preserve==0 || pMem->flags&(MEM_Blob|MEM_Str) ); + if( n<32 ) n = 32; if( sqlite3DbMallocSize(pMem->db, pMem->zMalloc)<n ){ if( preserve && pMem->z==pMem->zMalloc ){ diff --git a/lib/libsqlite3/src/vtab.c b/lib/libsqlite3/src/vtab.c index c7221cd3b7e..c561f7198f8 100644 --- a/lib/libsqlite3/src/vtab.c +++ b/lib/libsqlite3/src/vtab.c @@ -447,7 +447,7 @@ static int vtabCallConstructor( int (*xConstruct)(sqlite3*,void*,int,const char*const*,sqlite3_vtab**,char**), char **pzErr ){ - VtabCtx sCtx; + VtabCtx sCtx, *pPriorCtx; VTable *pVTable; int rc; const char *const*azArg = (const char *const*)pTab->azModuleArg; @@ -472,9 +472,10 @@ static int vtabCallConstructor( assert( xConstruct ); sCtx.pTab = pTab; sCtx.pVTable = pVTable; + pPriorCtx = db->pVtabCtx; db->pVtabCtx = &sCtx; rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr); - db->pVtabCtx = 0; + db->pVtabCtx = pPriorCtx; if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; if( SQLITE_OK!=rc ){ diff --git a/lib/libsqlite3/src/where.c b/lib/libsqlite3/src/where.c index e6083f0033e..d324228c9fa 100644 --- a/lib/libsqlite3/src/where.c +++ b/lib/libsqlite3/src/where.c @@ -686,7 +686,10 @@ static int isLikeOrGlob( #endif pList = pExpr->x.pList; pLeft = pList->a[1].pExpr; - if( pLeft->op!=TK_COLUMN || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT ){ + if( pLeft->op!=TK_COLUMN + || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT + || IsVirtual(pLeft->pTab) + ){ /* IMP: R-02065-49465 The left-hand side of the LIKE or GLOB operator must ** be the name of an indexed column with TEXT affinity. */ return 0; @@ -1559,15 +1562,19 @@ static int isDistinctRedundant( ** list, or else the WHERE clause contains a term of the form "col=X", ** where X is a constant value. The collation sequences of the ** comparison and select-list expressions must match those of the index. + ** + ** 3. All of those index columns for which the WHERE clause does not + ** contain a "col=X" term are subject to a NOT NULL constraint. */ for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ if( pIdx->onError==OE_None ) continue; for(i=0; i<pIdx->nColumn; i++){ int iCol = pIdx->aiColumn[i]; - if( 0==findTerm(pWC, iBase, iCol, ~(Bitmask)0, WO_EQ, pIdx) - && 0>findIndexCol(pParse, pDistinct, iBase, pIdx, i) - ){ - break; + if( 0==findTerm(pWC, iBase, iCol, ~(Bitmask)0, WO_EQ, pIdx) ){ + int iIdxCol = findIndexCol(pParse, pDistinct, iBase, pIdx, i); + if( iIdxCol<0 || pTab->aCol[pIdx->aiColumn[i]].notNull==0 ){ + break; + } } } if( i==pIdx->nColumn ){ @@ -1715,14 +1722,25 @@ static int isSortingIndex( } if( pIdx->onError!=OE_None && i==pIdx->nColumn && (wsFlags & WHERE_COLUMN_NULL)==0 - && !referencesOtherTables(pOrderBy, pMaskSet, j, base) ){ - /* All terms of this index match some prefix of the ORDER BY clause - ** and the index is UNIQUE and no terms on the tail of the ORDER BY - ** clause reference other tables in a join. If this is all true then - ** the order by clause is superfluous. Not that if the matching - ** condition is IS NULL then the result is not necessarily unique - ** even on a UNIQUE index, so disallow those cases. */ - return 1; + && !referencesOtherTables(pOrderBy, pMaskSet, j, base) + ){ + Column *aCol = pIdx->pTable->aCol; + + /* All terms of this index match some prefix of the ORDER BY clause, + ** the index is UNIQUE, and no terms on the tail of the ORDER BY + ** refer to other tables in a join. So, assuming that the index entries + ** visited contain no NULL values, then this index delivers rows in + ** the required order. + ** + ** It is not possible for any of the first nEqCol index fields to be + ** NULL (since the corresponding "=" operator in the WHERE clause would + ** not be true). So if all remaining index columns have NOT NULL + ** constaints attached to them, we can be confident that the visited + ** index entries are free of NULLs. */ + for(i=nEqCol; i<pIdx->nColumn; i++){ + if( aCol[pIdx->aiColumn[i]].notNull==0 ) break; + } + return (i==pIdx->nColumn); } return 0; } @@ -4383,7 +4401,7 @@ static Bitmask codeOneLoopStart( int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); int r; r = sqlite3ExprCodeGetColumn(pParse, pTabItem->pTab, -1, iCur, - regRowid); + regRowid, 0); sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, sqlite3VdbeCurrentAddr(v)+2, r, iSet); } diff --git a/lib/libsqlite3/tool/build-shell.sh b/lib/libsqlite3/tool/build-shell.sh index 54e83080954..8e62a71746c 100644 --- a/lib/libsqlite3/tool/build-shell.sh +++ b/lib/libsqlite3/tool/build-shell.sh @@ -12,8 +12,8 @@ make sqlite3.c gcc -o sqlite3 -g -Os -I. \ -DSQLITE_THREADSAFE=0 \ -DSQLITE_ENABLE_VFSTRACE \ - -DSQLITE_ENABLE_STAT2 \ - -DSQLITE_ENABLE_FTS3 \ + -DSQLITE_ENABLE_STAT3 \ + -DSQLITE_ENABLE_FTS4 \ -DSQLITE_ENABLE_RTREE \ -DHAVE_READLINE \ -DHAVE_USLEEP=1 \ diff --git a/lib/libsqlite3/tool/mkkeywordhash.c b/lib/libsqlite3/tool/mkkeywordhash.c index 509aeef9ea0..4e5ba8f1a8d 100644 --- a/lib/libsqlite3/tool/mkkeywordhash.c +++ b/lib/libsqlite3/tool/mkkeywordhash.c @@ -360,7 +360,7 @@ int main(int argc, char **argv){ /* Fill in the lengths of strings and hashes for all entries. */ for(i=0; i<nKeyword; i++){ Keyword *p = &aKeywordTable[i]; - p->len = strlen(p->zName); + p->len = (int)strlen(p->zName); assert( p->len<sizeof(p->zOrigName) ); strcpy(p->zOrigName, p->zName); totalLen += p->len; diff --git a/lib/libsqlite3/tool/showdb.c b/lib/libsqlite3/tool/showdb.c index 057abd32bae..d378d05d367 100644 --- a/lib/libsqlite3/tool/showdb.c +++ b/lib/libsqlite3/tool/showdb.c @@ -9,6 +9,7 @@ #include <unistd.h> #include <stdlib.h> #include <string.h> +#include "sqlite3.h" static int pagesize = 1024; /* Size of a database page */ @@ -451,6 +452,224 @@ static void decode_trunk_page( } /* +** A short text comment on the use of each page. +*/ +static char **zPageUse; + +/* +** Add a comment on the use of a page. +*/ +static void page_usage_msg(int pgno, const char *zFormat, ...){ + va_list ap; + char *zMsg; + + va_start(ap, zFormat); + zMsg = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + if( pgno<=0 || pgno>mxPage ){ + printf("ERROR: page %d out of bounds. Range=1..%d. Msg: %s\n", + pgno, mxPage, zMsg); + sqlite3_free(zMsg); + return; + } + if( zPageUse[pgno]!=0 ){ + printf("ERROR: page %d used multiple times:\n", pgno); + printf("ERROR: previous: %s\n", zPageUse[pgno]); + printf("ERROR: current: %s\n", zPageUse[pgno]); + sqlite3_free(zPageUse[pgno]); + } + zPageUse[pgno] = zMsg; +} + +/* +** Find overflow pages of a cell and describe their usage. +*/ +static void page_usage_cell( + unsigned char cType, /* Page type */ + unsigned char *a, /* Cell content */ + int pgno, /* page containing the cell */ + int cellno /* Index of the cell on the page */ +){ + int i; + int nDesc = 0; + int n = 0; + i64 nPayload; + i64 rowid; + int nLocal; + i = 0; + if( cType<=5 ){ + a += 4; + n += 4; + } + if( cType!=5 ){ + i = decodeVarint(a, &nPayload); + a += i; + n += i; + nLocal = localPayload(nPayload, cType); + }else{ + nPayload = nLocal = 0; + } + if( cType==5 || cType==13 ){ + i = decodeVarint(a, &rowid); + a += i; + n += i; + } + if( nLocal<nPayload ){ + int ovfl = decodeInt32(a+nLocal); + int cnt = 0; + while( ovfl && (cnt++)<mxPage ){ + page_usage_msg(ovfl, "overflow %d from cell %d of page %d", + cnt, cellno, pgno); + a = getContent((ovfl-1)*pagesize, 4); + ovfl = decodeInt32(a); + free(a); + } + } +} + + +/* +** Describe the usages of a b-tree page +*/ +static void page_usage_btree( + int pgno, /* Page to describe */ + int parent, /* Parent of this page. 0 for root pages */ + int idx, /* Which child of the parent */ + const char *zName /* Name of the table */ +){ + unsigned char *a; + const char *zType = "corrupt node"; + int nCell; + int i; + int hdr = pgno==1 ? 100 : 0; + + if( pgno<=0 || pgno>mxPage ) return; + a = getContent((pgno-1)*pagesize, pagesize); + switch( a[hdr] ){ + case 2: zType = "interior node of index"; break; + case 5: zType = "interior node of table"; break; + case 10: zType = "leaf of index"; break; + case 13: zType = "leaf of table"; break; + } + if( parent ){ + page_usage_msg(pgno, "%s [%s], child %d of page %d", + zType, zName, idx, parent); + }else{ + page_usage_msg(pgno, "root %s [%s]", zType, zName); + } + nCell = a[hdr+3]*256 + a[hdr+4]; + if( a[hdr]==2 || a[hdr]==5 ){ + int cellstart = hdr+12; + unsigned int child; + for(i=0; i<nCell; i++){ + int ofst; + + ofst = cellstart + i*2; + ofst = a[ofst]*256 + a[ofst+1]; + child = decodeInt32(a+ofst); + page_usage_btree(child, pgno, i, zName); + } + child = decodeInt32(a+cellstart-4); + page_usage_btree(child, pgno, i, zName); + } + if( a[hdr]==2 || a[hdr]==10 || a[hdr]==13 ){ + int cellstart = hdr + 8 + 4*(a[hdr]<=5); + for(i=0; i<nCell; i++){ + int ofst; + ofst = cellstart + i*2; + ofst = a[ofst]*256 + a[ofst+1]; + page_usage_cell(a[hdr], a+ofst, pgno, i); + } + } + free(a); +} + +/* +** Determine page usage by the freelist +*/ +static void page_usage_freelist(int pgno){ + unsigned char *a; + int cnt = 0; + int i; + int n; + int iNext; + int parent = 1; + + while( pgno>0 && pgno<=mxPage && (cnt++)<mxPage ){ + page_usage_msg(pgno, "freelist trunk #%d child of %d", cnt, parent); + a = getContent((pgno-1)*pagesize, pagesize); + iNext = decodeInt32(a); + n = decodeInt32(a+4); + for(i=0; i<n; i++){ + int child = decodeInt32(a + (i*4+8)); + page_usage_msg(child, "freelist leaf, child %d of trunk page %d", + i, pgno); + } + free(a); + parent = pgno; + pgno = iNext; + } +} + +/* +** Try to figure out how every page in the database file is being used. +*/ +static void page_usage_report(const char *zDbName){ + int i; + int rc; + sqlite3 *db; + sqlite3_stmt *pStmt; + unsigned char *a; + + /* Avoid the pathological case */ + if( mxPage<1 ){ + printf("empty database\n"); + return; + } + + /* Open the database file */ + rc = sqlite3_open(zDbName, &db); + if( rc ){ + printf("cannot open database: %s\n", sqlite3_errmsg(db)); + sqlite3_close(db); + return; + } + + /* Set up global variables zPageUse[] and mxPage to record page + ** usages */ + zPageUse = sqlite3_malloc( sizeof(zPageUse[0])*(mxPage+1) ); + if( zPageUse==0 ) out_of_memory(); + memset(zPageUse, 0, sizeof(zPageUse[0])*(mxPage+1)); + + /* Discover the usage of each page */ + a = getContent(0, 100); + page_usage_freelist(decodeInt32(a+32)); + free(a); + page_usage_btree(1, 0, 0, "sqlite_master"); + rc = sqlite3_prepare_v2(db, + "SELECT type, name, rootpage FROM SQLITE_MASTER WHERE rootpage", + -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + int pgno = sqlite3_column_int(pStmt, 2); + page_usage_btree(pgno, 0, 0, sqlite3_column_text(pStmt, 1)); + } + }else{ + printf("ERROR: cannot query database: %s\n", sqlite3_errmsg(db)); + } + sqlite3_finalize(pStmt); + sqlite3_close(db); + + /* Print the report and free memory used */ + for(i=1; i<=mxPage; i++){ + printf("%5d: %s\n", i, zPageUse[i] ? zPageUse[i] : "???"); + sqlite3_free(zPageUse[i]); + } + sqlite3_free(zPageUse); + zPageUse = 0; +} + +/* ** Print a usage comment */ static void usage(const char *argv0){ @@ -458,6 +677,7 @@ static void usage(const char *argv0){ fprintf(stderr, "args:\n" " dbheader Show database header\n" + " pgidx Index of how each page is used\n" " NNN..MMM Show hex of pages NNN through MMM\n" " NNN..end Show hex of pages NNN through end of file\n" " NNNb Decode btree page NNN\n" @@ -503,6 +723,10 @@ int main(int argc, char **argv){ print_db_header(); continue; } + if( strcmp(argv[i], "pgidx")==0 ){ + page_usage_report(argv[1]); + continue; + } if( !isdigit(argv[i][0]) ){ fprintf(stderr, "%s: unknown option: [%s]\n", argv[0], argv[i]); continue; diff --git a/lib/libsqlite3/tool/spaceanal.tcl b/lib/libsqlite3/tool/spaceanal.tcl index 3dddfe42d45..fd59670adae 100644 --- a/lib/libsqlite3/tool/spaceanal.tcl +++ b/lib/libsqlite3/tool/spaceanal.tcl @@ -49,15 +49,14 @@ if {$true_file_size<512} { # set extension [file extension $file_to_analyze] set pattern $file_to_analyze -append pattern {[0-9][0-9]} +append pattern {[0-3][0-9][0-9]} foreach f [glob -nocomplain $pattern] { incr true_file_size [file size $f] set extension {} } if {[string length $extension]>=2 && [string length $extension]<=4} { set pattern [file rootname $file_to_analyze] - append pattern [string range $extension 0 1] - append pattern {[0-9][0-9]} + append pattern {.[0-3][0-9][0-9]} foreach f [glob -nocomplain $pattern] { incr true_file_size [file size $f] } diff --git a/lib/libsqlite3/tool/warnings-clang.sh b/lib/libsqlite3/tool/warnings-clang.sh index 51084f31a63..b0d2fb6d108 100644 --- a/lib/libsqlite3/tool/warnings-clang.sh +++ b/lib/libsqlite3/tool/warnings-clang.sh @@ -9,5 +9,6 @@ echo '************* FTS4 and RTREE ****************' scan-build gcc -c -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE \ -DSQLITE_DEBUG sqlite3.c 2>&1 | grep -v 'ANALYZE:' echo '********** ENABLE_STAT3. THREADSAFE=0 *******' -scan-build gcc -c -DSQLITE_ENABLE_STAT3 -DSQLITE_THREADSAFE=0 \ - -DSQLITE_DEBUG sqlite3.c 2>&1 | grep -v 'ANALYZE:' +scan-build gcc -c -I. -DSQLITE_ENABLE_STAT3 -DSQLITE_THREADSAFE=0 \ + -DSQLITE_DEBUG \ + sqlite3.c ../sqlite/src/shell.c -ldl 2>&1 | grep -v 'ANALYZE:' |