1 This is a backport of the following sqlite changes: 2 3 1. http://sqlite.org/src/ci/9109128cb5 4 2. http://sqlite.org/src/ci/713b1b7dc1 5 3. http://sqlite.org/src/ci/8d1b5c3ac0 6 4. http://sqlite.org/src/ci/6b236069e1 7 5. http://sqlite.org/src/ci/880b51150a 8 9 which are needed for experiments with using unpatched sqlite. 10 If you hit a merge conflict on this file it is most likely 11 that you've upgraded to version of sqlite that includes those patches. 12 diff --git a/third_party/sqlite/amalgamation/sqlite3.c b/third_party/sqlite/amalgamation/sqlite3.c 13 index 3e794a9..73ff15f 100644 14 --- a/third_party/sqlite/amalgamation/sqlite3.c 15 +++ b/third_party/sqlite/amalgamation/sqlite3.c 16 @@ -24144,7 +24144,6 @@ struct unixFile { 17 sqlite3_io_methods const *pMethod; /* Always the first entry */ 18 unixInodeInfo *pInode; /* Info about locks on this inode */ 19 int h; /* The file descriptor */ 20 - int dirfd; /* File descriptor for the directory */ 21 unsigned char eFileLock; /* The type of lock held on this fd */ 22 unsigned char ctrlFlags; /* Behavioral bits. UNIXFILE_* flags */ 23 int lastErrno; /* The unix errno from last I/O error */ 24 @@ -24188,6 +24187,7 @@ struct unixFile { 25 */ 26 #define UNIXFILE_EXCL 0x01 /* Connections from one process only */ 27 #define UNIXFILE_RDONLY 0x02 /* Connection is read only */ 28 +#define UNIXFILE_DIRSYNC 0x04 /* Directory sync needed */ 29 30 /* 31 ** Include code that is common to all os_*.c files 32 @@ -24426,6 +24426,9 @@ SQLITE_API int sqlite3_open_file_count = 0; 33 #define threadid 0 34 #endif 35 36 +/* Forward reference */ 37 +static int openDirectory(const char*, int*); 38 + 39 /* 40 ** Many system calls are accessed through pointer-to-functions so that 41 ** they may be overridden at runtime to facilitate fault injection during 42 @@ -24522,6 +24525,12 @@ static struct unix_syscall { 43 #endif 44 #define osFallocate ((int(*)(int,off_t,off_t))aSyscall[15].pCurrent) 45 46 + { "unlink", (sqlite3_syscall_ptr)unlink, 0 }, 47 +#define osUnlink ((int(*)(const char*))aSyscall[16].pCurrent) 48 + 49 + { "openDirectory", (sqlite3_syscall_ptr)openDirectory, 0 }, 50 +#define osOpenDirectory ((int(*)(const char*,int*))aSyscall[17].pCurrent) 51 + 52 }; /* End of the overrideable system calls */ 53 54 /* 55 @@ -25876,10 +25885,6 @@ static int unixUnlock(sqlite3_file *id, int eFileLock){ 56 */ 57 static int closeUnixFile(sqlite3_file *id){ 58 unixFile *pFile = (unixFile*)id; 59 - if( pFile->dirfd>=0 ){ 60 - robust_close(pFile, pFile->dirfd, __LINE__); 61 - pFile->dirfd=-1; 62 - } 63 if( pFile->h>=0 ){ 64 robust_close(pFile, pFile->h, __LINE__); 65 pFile->h = -1; 66 @@ -25887,7 +25892,7 @@ static int closeUnixFile(sqlite3_file *id){ 67 #if OS_VXWORKS 68 if( pFile->pId ){ 69 if( pFile->isDelete ){ 70 - unlink(pFile->pId->zCanonicalName); 71 + osUnlink(pFile->pId->zCanonicalName); 72 } 73 vxworksReleaseFileId(pFile->pId); 74 pFile->pId = 0; 75 @@ -26134,7 +26139,7 @@ static int dotlockUnlock(sqlite3_file *id, int eFileLock) { 76 77 /* To fully unlock the database, delete the lock file */ 78 assert( eFileLock==NO_LOCK ); 79 - if( unlink(zLockFile) ){ 80 + if( osUnlink(zLockFile) ){ 81 int rc = 0; 82 int tErrno = errno; 83 if( ENOENT != tErrno ){ 84 @@ -27371,6 +27376,50 @@ static int full_fsync(int fd, int fullSync, int dataOnly){ 85 } 86 87 /* 88 +** Open a file descriptor to the directory containing file zFilename. 89 +** If successful, *pFd is set to the opened file descriptor and 90 +** SQLITE_OK is returned. If an error occurs, either SQLITE_NOMEM 91 +** or SQLITE_CANTOPEN is returned and *pFd is set to an undefined 92 +** value. 93 +** 94 +** The directory file descriptor is used for only one thing - to 95 +** fsync() a directory to make sure file creation and deletion events 96 +** are flushed to disk. Such fsyncs are not needed on newer 97 +** journaling filesystems, but are required on older filesystems. 98 +** 99 +** This routine can be overridden using the xSetSysCall interface. 100 +** The ability to override this routine was added in support of the 101 +** chromium sandbox. Opening a directory is a security risk (we are 102 +** told) so making it overrideable allows the chromium sandbox to 103 +** replace this routine with a harmless no-op. To make this routine 104 +** a no-op, replace it with a stub that returns SQLITE_OK but leaves 105 +** *pFd set to a negative number. 106 +** 107 +** If SQLITE_OK is returned, the caller is responsible for closing 108 +** the file descriptor *pFd using close(). 109 +*/ 110 +static int openDirectory(const char *zFilename, int *pFd){ 111 + int ii; 112 + int fd = -1; 113 + char zDirname[MAX_PATHNAME+1]; 114 + 115 + sqlite3_snprintf(MAX_PATHNAME, zDirname, "%s", zFilename); 116 + for(ii=(int)strlen(zDirname); ii>1 && zDirname[ii]!='/'; ii--); 117 + if( ii>0 ){ 118 + zDirname[ii] = '\0'; 119 + fd = robust_open(zDirname, O_RDONLY|O_BINARY, 0); 120 + if( fd>=0 ){ 121 +#ifdef FD_CLOEXEC 122 + osFcntl(fd, F_SETFD, osFcntl(fd, F_GETFD, 0) | FD_CLOEXEC); 123 +#endif 124 + OSTRACE(("OPENDIR %-3d %s\n", fd, zDirname)); 125 + } 126 + } 127 + *pFd = fd; 128 + return (fd>=0?SQLITE_OK:unixLogError(SQLITE_CANTOPEN_BKPT, "open", zDirname)); 129 +} 130 + 131 +/* 132 ** Make sure all writes to a particular file are committed to disk. 133 ** 134 ** If dataOnly==0 then both the file itself and its metadata (file 135 @@ -27410,28 +27459,21 @@ static int unixSync(sqlite3_file *id, int flags){ 136 pFile->lastErrno = errno; 137 return unixLogError(SQLITE_IOERR_FSYNC, "full_fsync", pFile->zPath); 138 } 139 - if( pFile->dirfd>=0 ){ 140 - OSTRACE(("DIRSYNC %-3d (have_fullfsync=%d fullsync=%d)\n", pFile->dirfd, 141 + 142 + /* Also fsync the directory containing the file if the DIRSYNC flag 143 + ** is set. This is a one-time occurrance. Many systems (examples: AIX) 144 + ** are unable to fsync a directory, so ignore errors on the fsync. 145 + */ 146 + if( pFile->ctrlFlags & UNIXFILE_DIRSYNC ){ 147 + int dirfd; 148 + OSTRACE(("DIRSYNC %s (have_fullfsync=%d fullsync=%d)\n", pFile->zPath, 149 HAVE_FULLFSYNC, isFullsync)); 150 -#ifndef SQLITE_DISABLE_DIRSYNC 151 - /* The directory sync is only attempted if full_fsync is 152 - ** turned off or unavailable. If a full_fsync occurred above, 153 - ** then the directory sync is superfluous. 154 - */ 155 - if( (!HAVE_FULLFSYNC || !isFullsync) && full_fsync(pFile->dirfd,0,0) ){ 156 - /* 157 - ** We have received multiple reports of fsync() returning 158 - ** errors when applied to directories on certain file systems. 159 - ** A failed directory sync is not a big deal. So it seems 160 - ** better to ignore the error. Ticket #1657 161 - */ 162 - /* pFile->lastErrno = errno; */ 163 - /* return SQLITE_IOERR; */ 164 + rc = osOpenDirectory(pFile->zPath, &dirfd); 165 + if( rc==SQLITE_OK && dirfd>=0 ){ 166 + full_fsync(dirfd, 0, 0); 167 + robust_close(pFile, dirfd, __LINE__); 168 } 169 -#endif 170 - /* Only need to sync once, so close the directory when we are done */ 171 - robust_close(pFile, pFile->dirfd, __LINE__); 172 - pFile->dirfd = -1; 173 + pFile->ctrlFlags &= ~UNIXFILE_DIRSYNC; 174 } 175 return rc; 176 } 177 @@ -28255,7 +28297,7 @@ static int unixShmUnmap( 178 assert( pShmNode->nRef>0 ); 179 pShmNode->nRef--; 180 if( pShmNode->nRef==0 ){ 181 - if( deleteFlag && pShmNode->h>=0 ) unlink(pShmNode->zFilename); 182 + if( deleteFlag && pShmNode->h>=0 ) osUnlink(pShmNode->zFilename); 183 unixShmPurge(pDbFd); 184 } 185 unixLeaveMutex(); 186 @@ -28575,7 +28617,7 @@ void initUnixFile(sqlite3_file* file) { 187 int fillInUnixFile( 188 sqlite3_vfs *pVfs, /* Pointer to vfs object */ 189 int h, /* Open file descriptor of file being opened */ 190 - int dirfd, /* Directory file descriptor */ 191 + int syncDir, /* True to sync directory on first sync */ 192 sqlite3_file *pId, /* Write to the unixFile structure here */ 193 const char *zFilename, /* Name of the file being opened */ 194 int noLock, /* Omit locking if true */ 195 @@ -28606,7 +28648,6 @@ int fillInUnixFile( 196 197 OSTRACE(("OPEN %-3d %s\n", h, zFilename)); 198 pNew->h = h; 199 - pNew->dirfd = dirfd; 200 pNew->zPath = zFilename; 201 if( memcmp(pVfs->zName,"unix-excl",10)==0 ){ 202 pNew->ctrlFlags = UNIXFILE_EXCL; 203 @@ -28616,6 +28657,9 @@ int fillInUnixFile( 204 if( isReadOnly ){ 205 pNew->ctrlFlags |= UNIXFILE_RDONLY; 206 } 207 + if( syncDir ){ 208 + pNew->ctrlFlags |= UNIXFILE_DIRSYNC; 209 + } 210 211 #if OS_VXWORKS 212 pNew->pId = vxworksFindFileId(zFilename); 213 @@ -28742,13 +28786,12 @@ int fillInUnixFile( 214 if( rc!=SQLITE_OK ){ 215 if( h>=0 ) robust_close(pNew, h, __LINE__); 216 h = -1; 217 - unlink(zFilename); 218 + osUnlink(zFilename); 219 isDelete = 0; 220 } 221 pNew->isDelete = isDelete; 222 #endif 223 if( rc!=SQLITE_OK ){ 224 - if( dirfd>=0 ) robust_close(pNew, dirfd, __LINE__); 225 if( h>=0 ) robust_close(pNew, h, __LINE__); 226 }else{ 227 pNew->pMethod = pLockingStyle; 228 @@ -28758,37 +28801,6 @@ int fillInUnixFile( 229 } 230 231 /* 232 -** Open a file descriptor to the directory containing file zFilename. 233 -** If successful, *pFd is set to the opened file descriptor and 234 -** SQLITE_OK is returned. If an error occurs, either SQLITE_NOMEM 235 -** or SQLITE_CANTOPEN is returned and *pFd is set to an undefined 236 -** value. 237 -** 238 -** If SQLITE_OK is returned, the caller is responsible for closing 239 -** the file descriptor *pFd using close(). 240 -*/ 241 -static int openDirectory(const char *zFilename, int *pFd){ 242 - int ii; 243 - int fd = -1; 244 - char zDirname[MAX_PATHNAME+1]; 245 - 246 - sqlite3_snprintf(MAX_PATHNAME, zDirname, "%s", zFilename); 247 - for(ii=(int)strlen(zDirname); ii>1 && zDirname[ii]!='/'; ii--); 248 - if( ii>0 ){ 249 - zDirname[ii] = '\0'; 250 - fd = robust_open(zDirname, O_RDONLY|O_BINARY, 0); 251 - if( fd>=0 ){ 252 -#ifdef FD_CLOEXEC 253 - osFcntl(fd, F_SETFD, osFcntl(fd, F_GETFD, 0) | FD_CLOEXEC); 254 -#endif 255 - OSTRACE(("OPENDIR %-3d %s\n", fd, zDirname)); 256 - } 257 - } 258 - *pFd = fd; 259 - return (fd>=0?SQLITE_OK:unixLogError(SQLITE_CANTOPEN_BKPT, "open", zDirname)); 260 -} 261 - 262 -/* 263 ** Return the name of a directory in which to put temporary files. 264 ** If no suitable temporary file directory can be found, return NULL. 265 */ 266 @@ -29083,7 +29095,6 @@ static int unixOpen( 267 ){ 268 unixFile *p = (unixFile *)pFile; 269 int fd = -1; /* File descriptor returned by open() */ 270 - int dirfd = -1; /* Directory file descriptor */ 271 int openFlags = 0; /* Flags to pass to open() */ 272 int eType = flags&0xFFFFFF00; /* Type of file to open */ 273 int noLock; /* True to omit locking primitives */ 274 @@ -29102,7 +29113,7 @@ static int unixOpen( 275 ** a file-descriptor on the directory too. The first time unixSync() 276 ** is called the directory file descriptor will be fsync()ed and close()d. 277 */ 278 - int isOpenDirectory = (isCreate && ( 279 + int syncDir = (isCreate && ( 280 eType==SQLITE_OPEN_MASTER_JOURNAL 281 || eType==SQLITE_OPEN_MAIN_JOURNAL 282 || eType==SQLITE_OPEN_WAL 283 @@ -29149,7 +29160,7 @@ static int unixOpen( 284 } 285 }else if( !zName ){ 286 /* If zName is NULL, the upper layer is requesting a temp file. */ 287 - assert(isDelete && !isOpenDirectory); 288 + assert(isDelete && !syncDir); 289 rc = unixGetTempname(MAX_PATHNAME+1, zTmpname); 290 if( rc!=SQLITE_OK ){ 291 return rc; 292 @@ -29202,7 +29213,7 @@ static int unixOpen( 293 #if OS_VXWORKS 294 zPath = zName; 295 #else 296 - unlink(zName); 297 + osUnlink(zName); 298 #endif 299 } 300 #if SQLITE_ENABLE_LOCKING_STYLE 301 @@ -29211,19 +29222,6 @@ static int unixOpen( 302 } 303 #endif 304 305 - if( isOpenDirectory ){ 306 - rc = openDirectory(zPath, &dirfd); 307 - if( rc!=SQLITE_OK ){ 308 - /* It is safe to close fd at this point, because it is guaranteed not 309 - ** to be open on a database file. If it were open on a database file, 310 - ** it would not be safe to close as this would release any locks held 311 - ** on the file by this process. */ 312 - assert( eType!=SQLITE_OPEN_MAIN_DB ); 313 - robust_close(p, fd, __LINE__); 314 - goto open_finished; 315 - } 316 - } 317 - 318 #ifdef FD_CLOEXEC 319 osFcntl(fd, F_SETFD, osFcntl(fd, F_GETFD, 0) | FD_CLOEXEC); 320 #endif 321 @@ -29235,7 +29233,6 @@ static int unixOpen( 322 struct statfs fsInfo; 323 if( fstatfs(fd, &fsInfo) == -1 ){ 324 ((unixFile*)pFile)->lastErrno = errno; 325 - if( dirfd>=0 ) robust_close(p, dirfd, __LINE__); 326 robust_close(p, fd, __LINE__); 327 return SQLITE_IOERR_ACCESS; 328 } 329 @@ -29267,9 +29264,6 @@ static int unixOpen( 330 ** not while other file descriptors opened by the same process on 331 ** the same file are working. */ 332 p->lastErrno = errno; 333 - if( dirfd>=0 ){ 334 - robust_close(p, dirfd, __LINE__); 335 - } 336 robust_close(p, fd, __LINE__); 337 rc = SQLITE_IOERR_ACCESS; 338 goto open_finished; 339 @@ -29277,7 +29271,7 @@ static int unixOpen( 340 useProxy = !(fsInfo.f_flags&MNT_LOCAL); 341 } 342 if( useProxy ){ 343 - rc = fillInUnixFile(pVfs, fd, dirfd, pFile, zPath, noLock, 344 + rc = fillInUnixFile(pVfs, fd, syncDir, pFile, zPath, noLock, 345 isDelete, isReadonly); 346 if( rc==SQLITE_OK ){ 347 rc = proxyTransformUnixFile((unixFile*)pFile, ":auto:"); 348 @@ -29295,7 +29289,7 @@ static int unixOpen( 349 } 350 #endif 351 352 - rc = fillInUnixFile(pVfs, fd, dirfd, pFile, zPath, noLock, 353 + rc = fillInUnixFile(pVfs, fd, syncDir, pFile, zPath, noLock, 354 isDelete, isReadonly); 355 open_finished: 356 if( rc!=SQLITE_OK ){ 357 @@ -29317,13 +29311,13 @@ static int unixDelete( 358 int rc = SQLITE_OK; 359 UNUSED_PARAMETER(NotUsed); 360 SimulateIOError(return SQLITE_IOERR_DELETE); 361 - if( unlink(zPath)==(-1) && errno!=ENOENT ){ 362 + if( osUnlink(zPath)==(-1) && errno!=ENOENT ){ 363 return unixLogError(SQLITE_IOERR_DELETE, "unlink", zPath); 364 } 365 #ifndef SQLITE_DISABLE_DIRSYNC 366 if( dirSync ){ 367 int fd; 368 - rc = openDirectory(zPath, &fd); 369 + rc = osOpenDirectory(zPath, &fd); 370 if( rc==SQLITE_OK ){ 371 #if OS_VXWORKS 372 if( fsync(fd)==-1 ) 373 @@ -29895,7 +29889,6 @@ static int proxyCreateUnixFile( 374 int islockfile /* if non zero missing dirs will be created */ 375 ) { 376 int fd = -1; 377 - int dirfd = -1; 378 unixFile *pNew; 379 int rc = SQLITE_OK; 380 int openFlags = O_RDWR | O_CREAT; 381 @@ -29960,7 +29953,7 @@ static int proxyCreateUnixFile( 382 pUnused->flags = openFlags; 383 pNew->pUnused = pUnused; 384 385 - rc = fillInUnixFile(&dummyVfs, fd, dirfd, (sqlite3_file*)pNew, path, 0, 0, 0); 386 + rc = fillInUnixFile(&dummyVfs, fd, 0, (sqlite3_file*)pNew, path, 0, 0, 0); 387 if( rc==SQLITE_OK ){ 388 *ppFile = pNew; 389 return SQLITE_OK; 390 @@ -30074,7 +30067,7 @@ static int proxyBreakConchLock(unixFile *pFile, uuid_t myHostID){ 391 end_breaklock: 392 if( rc ){ 393 if( fd>=0 ){ 394 - unlink(tPath); 395 + osUnlink(tPath); 396 robust_close(pFile, fd, __LINE__); 397 } 398 fprintf(stderr, "failed to break stale lock on %s, %s\n", cPath, errmsg); 399 @@ -30897,7 +30890,7 @@ SQLITE_API int sqlite3_os_init(void){ 400 401 /* Double-check that the aSyscall[] array has been constructed 402 ** correctly. See ticket [bb3a86e890c8e96ab] */ 403 - assert( ArraySize(aSyscall)==16 ); 404 + assert( ArraySize(aSyscall)==18 ); 405 406 /* Register all VFSes defined in the aVfs[] array */ 407 for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ 408 diff --git a/third_party/sqlite/src/src/os_unix.c b/third_party/sqlite/src/src/os_unix.c 409 index e5b2540..804c588 100644 410 --- a/third_party/sqlite/src/src/os_unix.c 411 +++ b/third_party/sqlite/src/src/os_unix.c 412 @@ -204,7 +204,6 @@ struct unixFile { 413 sqlite3_io_methods const *pMethod; /* Always the first entry */ 414 unixInodeInfo *pInode; /* Info about locks on this inode */ 415 int h; /* The file descriptor */ 416 - int dirfd; /* File descriptor for the directory */ 417 unsigned char eFileLock; /* The type of lock held on this fd */ 418 unsigned char ctrlFlags; /* Behavioral bits. UNIXFILE_* flags */ 419 int lastErrno; /* The unix errno from last I/O error */ 420 @@ -248,6 +247,7 @@ struct unixFile { 421 */ 422 #define UNIXFILE_EXCL 0x01 /* Connections from one process only */ 423 #define UNIXFILE_RDONLY 0x02 /* Connection is read only */ 424 +#define UNIXFILE_DIRSYNC 0x04 /* Directory sync needed */ 425 426 /* 427 ** Include code that is common to all os_*.c files 428 @@ -281,6 +281,9 @@ struct unixFile { 429 #define threadid 0 430 #endif 431 432 +/* Forward reference */ 433 +static int openDirectory(const char*, int*); 434 + 435 /* 436 ** Many system calls are accessed through pointer-to-functions so that 437 ** they may be overridden at runtime to facilitate fault injection during 438 @@ -377,6 +380,12 @@ static struct unix_syscall { 439 #endif 440 #define osFallocate ((int(*)(int,off_t,off_t))aSyscall[15].pCurrent) 441 442 + { "unlink", (sqlite3_syscall_ptr)unlink, 0 }, 443 +#define osUnlink ((int(*)(const char*))aSyscall[16].pCurrent) 444 + 445 + { "openDirectory", (sqlite3_syscall_ptr)openDirectory, 0 }, 446 +#define osOpenDirectory ((int(*)(const char*,int*))aSyscall[17].pCurrent) 447 + 448 }; /* End of the overrideable system calls */ 449 450 /* 451 @@ -1731,10 +1740,6 @@ static int unixUnlock(sqlite3_file *id, int eFileLock){ 452 */ 453 static int closeUnixFile(sqlite3_file *id){ 454 unixFile *pFile = (unixFile*)id; 455 - if( pFile->dirfd>=0 ){ 456 - robust_close(pFile, pFile->dirfd, __LINE__); 457 - pFile->dirfd=-1; 458 - } 459 if( pFile->h>=0 ){ 460 robust_close(pFile, pFile->h, __LINE__); 461 pFile->h = -1; 462 @@ -1742,7 +1747,7 @@ static int closeUnixFile(sqlite3_file *id){ 463 #if OS_VXWORKS 464 if( pFile->pId ){ 465 if( pFile->isDelete ){ 466 - unlink(pFile->pId->zCanonicalName); 467 + osUnlink(pFile->pId->zCanonicalName); 468 } 469 vxworksReleaseFileId(pFile->pId); 470 pFile->pId = 0; 471 @@ -1989,7 +1994,7 @@ static int dotlockUnlock(sqlite3_file *id, int eFileLock) { 472 473 /* To fully unlock the database, delete the lock file */ 474 assert( eFileLock==NO_LOCK ); 475 - if( unlink(zLockFile) ){ 476 + if( osUnlink(zLockFile) ){ 477 int rc = 0; 478 int tErrno = errno; 479 if( ENOENT != tErrno ){ 480 @@ -3226,6 +3231,50 @@ static int full_fsync(int fd, int fullSync, int dataOnly){ 481 } 482 483 /* 484 +** Open a file descriptor to the directory containing file zFilename. 485 +** If successful, *pFd is set to the opened file descriptor and 486 +** SQLITE_OK is returned. If an error occurs, either SQLITE_NOMEM 487 +** or SQLITE_CANTOPEN is returned and *pFd is set to an undefined 488 +** value. 489 +** 490 +** The directory file descriptor is used for only one thing - to 491 +** fsync() a directory to make sure file creation and deletion events 492 +** are flushed to disk. Such fsyncs are not needed on newer 493 +** journaling filesystems, but are required on older filesystems. 494 +** 495 +** This routine can be overridden using the xSetSysCall interface. 496 +** The ability to override this routine was added in support of the 497 +** chromium sandbox. Opening a directory is a security risk (we are 498 +** told) so making it overrideable allows the chromium sandbox to 499 +** replace this routine with a harmless no-op. To make this routine 500 +** a no-op, replace it with a stub that returns SQLITE_OK but leaves 501 +** *pFd set to a negative number. 502 +** 503 +** If SQLITE_OK is returned, the caller is responsible for closing 504 +** the file descriptor *pFd using close(). 505 +*/ 506 +static int openDirectory(const char *zFilename, int *pFd){ 507 + int ii; 508 + int fd = -1; 509 + char zDirname[MAX_PATHNAME+1]; 510 + 511 + sqlite3_snprintf(MAX_PATHNAME, zDirname, "%s", zFilename); 512 + for(ii=(int)strlen(zDirname); ii>1 && zDirname[ii]!='/'; ii--); 513 + if( ii>0 ){ 514 + zDirname[ii] = '\0'; 515 + fd = robust_open(zDirname, O_RDONLY|O_BINARY, 0); 516 + if( fd>=0 ){ 517 +#ifdef FD_CLOEXEC 518 + osFcntl(fd, F_SETFD, osFcntl(fd, F_GETFD, 0) | FD_CLOEXEC); 519 +#endif 520 + OSTRACE(("OPENDIR %-3d %s\n", fd, zDirname)); 521 + } 522 + } 523 + *pFd = fd; 524 + return (fd>=0?SQLITE_OK:unixLogError(SQLITE_CANTOPEN_BKPT, "open", zDirname)); 525 +} 526 + 527 +/* 528 ** Make sure all writes to a particular file are committed to disk. 529 ** 530 ** If dataOnly==0 then both the file itself and its metadata (file 531 @@ -3265,28 +3314,23 @@ static int unixSync(sqlite3_file *id, int flags){ 532 pFile->lastErrno = errno; 533 return unixLogError(SQLITE_IOERR_FSYNC, "full_fsync", pFile->zPath); 534 } 535 - if( pFile->dirfd>=0 ){ 536 - OSTRACE(("DIRSYNC %-3d (have_fullfsync=%d fullsync=%d)\n", pFile->dirfd, 537 + 538 + /* Also fsync the directory containing the file if the DIRSYNC flag 539 + ** is set. This is a one-time occurrance. Many systems (examples: AIX) 540 + ** are unable to fsync a directory, so ignore errors on the fsync. 541 + */ 542 + if( pFile->ctrlFlags & UNIXFILE_DIRSYNC ){ 543 + int dirfd; 544 + OSTRACE(("DIRSYNC %s (have_fullfsync=%d fullsync=%d)\n", pFile->zPath, 545 HAVE_FULLFSYNC, isFullsync)); 546 -#ifndef SQLITE_DISABLE_DIRSYNC 547 - /* The directory sync is only attempted if full_fsync is 548 - ** turned off or unavailable. If a full_fsync occurred above, 549 - ** then the directory sync is superfluous. 550 - */ 551 - if( (!HAVE_FULLFSYNC || !isFullsync) && full_fsync(pFile->dirfd,0,0) ){ 552 - /* 553 - ** We have received multiple reports of fsync() returning 554 - ** errors when applied to directories on certain file systems. 555 - ** A failed directory sync is not a big deal. So it seems 556 - ** better to ignore the error. Ticket #1657 557 - */ 558 - /* pFile->lastErrno = errno; */ 559 - /* return SQLITE_IOERR; */ 560 + rc = osOpenDirectory(pFile->zPath, &dirfd); 561 + if( rc==SQLITE_OK && dirfd>=0 ){ 562 + full_fsync(dirfd, 0, 0); 563 + robust_close(pFile, dirfd, __LINE__); 564 + }else if( rc==SQLITE_CANTOPEN ){ 565 + rc = SQLITE_OK; 566 } 567 -#endif 568 - /* Only need to sync once, so close the directory when we are done */ 569 - robust_close(pFile, pFile->dirfd, __LINE__); 570 - pFile->dirfd = -1; 571 + pFile->ctrlFlags &= ~UNIXFILE_DIRSYNC; 572 } 573 return rc; 574 } 575 @@ -4110,7 +4154,7 @@ static int unixShmUnmap( 576 assert( pShmNode->nRef>0 ); 577 pShmNode->nRef--; 578 if( pShmNode->nRef==0 ){ 579 - if( deleteFlag && pShmNode->h>=0 ) unlink(pShmNode->zFilename); 580 + if( deleteFlag && pShmNode->h>=0 ) osUnlink(pShmNode->zFilename); 581 unixShmPurge(pDbFd); 582 } 583 unixLeaveMutex(); 584 @@ -4430,7 +4474,7 @@ void initUnixFile(sqlite3_file* file) { 585 int fillInUnixFile( 586 sqlite3_vfs *pVfs, /* Pointer to vfs object */ 587 int h, /* Open file descriptor of file being opened */ 588 - int dirfd, /* Directory file descriptor */ 589 + int syncDir, /* True to sync directory on first sync */ 590 sqlite3_file *pId, /* Write to the unixFile structure here */ 591 const char *zFilename, /* Name of the file being opened */ 592 int noLock, /* Omit locking if true */ 593 @@ -4461,7 +4505,6 @@ int fillInUnixFile( 594 595 OSTRACE(("OPEN %-3d %s\n", h, zFilename)); 596 pNew->h = h; 597 - pNew->dirfd = dirfd; 598 pNew->zPath = zFilename; 599 if( memcmp(pVfs->zName,"unix-excl",10)==0 ){ 600 pNew->ctrlFlags = UNIXFILE_EXCL; 601 @@ -4471,6 +4514,9 @@ int fillInUnixFile( 602 if( isReadOnly ){ 603 pNew->ctrlFlags |= UNIXFILE_RDONLY; 604 } 605 + if( syncDir ){ 606 + pNew->ctrlFlags |= UNIXFILE_DIRSYNC; 607 + } 608 609 #if OS_VXWORKS 610 pNew->pId = vxworksFindFileId(zFilename); 611 @@ -4597,13 +4643,12 @@ int fillInUnixFile( 612 if( rc!=SQLITE_OK ){ 613 if( h>=0 ) robust_close(pNew, h, __LINE__); 614 h = -1; 615 - unlink(zFilename); 616 + osUnlink(zFilename); 617 isDelete = 0; 618 } 619 pNew->isDelete = isDelete; 620 #endif 621 if( rc!=SQLITE_OK ){ 622 - if( dirfd>=0 ) robust_close(pNew, dirfd, __LINE__); 623 if( h>=0 ) robust_close(pNew, h, __LINE__); 624 }else{ 625 pNew->pMethod = pLockingStyle; 626 @@ -4613,37 +4658,6 @@ int fillInUnixFile( 627 } 628 629 /* 630 -** Open a file descriptor to the directory containing file zFilename. 631 -** If successful, *pFd is set to the opened file descriptor and 632 -** SQLITE_OK is returned. If an error occurs, either SQLITE_NOMEM 633 -** or SQLITE_CANTOPEN is returned and *pFd is set to an undefined 634 -** value. 635 -** 636 -** If SQLITE_OK is returned, the caller is responsible for closing 637 -** the file descriptor *pFd using close(). 638 -*/ 639 -static int openDirectory(const char *zFilename, int *pFd){ 640 - int ii; 641 - int fd = -1; 642 - char zDirname[MAX_PATHNAME+1]; 643 - 644 - sqlite3_snprintf(MAX_PATHNAME, zDirname, "%s", zFilename); 645 - for(ii=(int)strlen(zDirname); ii>1 && zDirname[ii]!='/'; ii--); 646 - if( ii>0 ){ 647 - zDirname[ii] = '\0'; 648 - fd = robust_open(zDirname, O_RDONLY|O_BINARY, 0); 649 - if( fd>=0 ){ 650 -#ifdef FD_CLOEXEC 651 - osFcntl(fd, F_SETFD, osFcntl(fd, F_GETFD, 0) | FD_CLOEXEC); 652 -#endif 653 - OSTRACE(("OPENDIR %-3d %s\n", fd, zDirname)); 654 - } 655 - } 656 - *pFd = fd; 657 - return (fd>=0?SQLITE_OK:unixLogError(SQLITE_CANTOPEN_BKPT, "open", zDirname)); 658 -} 659 - 660 -/* 661 ** Return the name of a directory in which to put temporary files. 662 ** If no suitable temporary file directory can be found, return NULL. 663 */ 664 @@ -4938,7 +4952,6 @@ static int unixOpen( 665 ){ 666 unixFile *p = (unixFile *)pFile; 667 int fd = -1; /* File descriptor returned by open() */ 668 - int dirfd = -1; /* Directory file descriptor */ 669 int openFlags = 0; /* Flags to pass to open() */ 670 int eType = flags&0xFFFFFF00; /* Type of file to open */ 671 int noLock; /* True to omit locking primitives */ 672 @@ -4957,7 +4970,7 @@ static int unixOpen( 673 ** a file-descriptor on the directory too. The first time unixSync() 674 ** is called the directory file descriptor will be fsync()ed and close()d. 675 */ 676 - int isOpenDirectory = (isCreate && ( 677 + int syncDir = (isCreate && ( 678 eType==SQLITE_OPEN_MASTER_JOURNAL 679 || eType==SQLITE_OPEN_MAIN_JOURNAL 680 || eType==SQLITE_OPEN_WAL 681 @@ -5004,7 +5017,7 @@ static int unixOpen( 682 } 683 }else if( !zName ){ 684 /* If zName is NULL, the upper layer is requesting a temp file. */ 685 - assert(isDelete && !isOpenDirectory); 686 + assert(isDelete && !syncDir); 687 rc = unixGetTempname(MAX_PATHNAME+1, zTmpname); 688 if( rc!=SQLITE_OK ){ 689 return rc; 690 @@ -5057,7 +5070,7 @@ static int unixOpen( 691 #if OS_VXWORKS 692 zPath = zName; 693 #else 694 - unlink(zName); 695 + osUnlink(zName); 696 #endif 697 } 698 #if SQLITE_ENABLE_LOCKING_STYLE 699 @@ -5066,19 +5079,6 @@ static int unixOpen( 700 } 701 #endif 702 703 - if( isOpenDirectory ){ 704 - rc = openDirectory(zPath, &dirfd); 705 - if( rc!=SQLITE_OK ){ 706 - /* It is safe to close fd at this point, because it is guaranteed not 707 - ** to be open on a database file. If it were open on a database file, 708 - ** it would not be safe to close as this would release any locks held 709 - ** on the file by this process. */ 710 - assert( eType!=SQLITE_OPEN_MAIN_DB ); 711 - robust_close(p, fd, __LINE__); 712 - goto open_finished; 713 - } 714 - } 715 - 716 #ifdef FD_CLOEXEC 717 osFcntl(fd, F_SETFD, osFcntl(fd, F_GETFD, 0) | FD_CLOEXEC); 718 #endif 719 @@ -5090,7 +5090,6 @@ static int unixOpen( 720 struct statfs fsInfo; 721 if( fstatfs(fd, &fsInfo) == -1 ){ 722 ((unixFile*)pFile)->lastErrno = errno; 723 - if( dirfd>=0 ) robust_close(p, dirfd, __LINE__); 724 robust_close(p, fd, __LINE__); 725 return SQLITE_IOERR_ACCESS; 726 } 727 @@ -5122,9 +5121,6 @@ static int unixOpen( 728 ** not while other file descriptors opened by the same process on 729 ** the same file are working. */ 730 p->lastErrno = errno; 731 - if( dirfd>=0 ){ 732 - robust_close(p, dirfd, __LINE__); 733 - } 734 robust_close(p, fd, __LINE__); 735 rc = SQLITE_IOERR_ACCESS; 736 goto open_finished; 737 @@ -5132,7 +5128,7 @@ static int unixOpen( 738 useProxy = !(fsInfo.f_flags&MNT_LOCAL); 739 } 740 if( useProxy ){ 741 - rc = fillInUnixFile(pVfs, fd, dirfd, pFile, zPath, noLock, 742 + rc = fillInUnixFile(pVfs, fd, syncDir, pFile, zPath, noLock, 743 isDelete, isReadonly); 744 if( rc==SQLITE_OK ){ 745 rc = proxyTransformUnixFile((unixFile*)pFile, ":auto:"); 746 @@ -5150,7 +5146,7 @@ static int unixOpen( 747 } 748 #endif 749 750 - rc = fillInUnixFile(pVfs, fd, dirfd, pFile, zPath, noLock, 751 + rc = fillInUnixFile(pVfs, fd, syncDir, pFile, zPath, noLock, 752 isDelete, isReadonly); 753 open_finished: 754 if( rc!=SQLITE_OK ){ 755 @@ -5172,13 +5168,13 @@ static int unixDelete( 756 int rc = SQLITE_OK; 757 UNUSED_PARAMETER(NotUsed); 758 SimulateIOError(return SQLITE_IOERR_DELETE); 759 - if( unlink(zPath)==(-1) && errno!=ENOENT ){ 760 + if( osUnlink(zPath)==(-1) && errno!=ENOENT ){ 761 return unixLogError(SQLITE_IOERR_DELETE, "unlink", zPath); 762 } 763 #ifndef SQLITE_DISABLE_DIRSYNC 764 if( dirSync ){ 765 int fd; 766 - rc = openDirectory(zPath, &fd); 767 + rc = osOpenDirectory(zPath, &fd); 768 if( rc==SQLITE_OK ){ 769 #if OS_VXWORKS 770 if( fsync(fd)==-1 ) 771 @@ -5189,6 +5185,8 @@ static int unixDelete( 772 rc = unixLogError(SQLITE_IOERR_DIR_FSYNC, "fsync", zPath); 773 } 774 robust_close(0, fd, __LINE__); 775 + }else if( rc==SQLITE_CANTOPEN ){ 776 + rc = SQLITE_OK; 777 } 778 } 779 #endif 780 @@ -5750,7 +5748,6 @@ static int proxyCreateUnixFile( 781 int islockfile /* if non zero missing dirs will be created */ 782 ) { 783 int fd = -1; 784 - int dirfd = -1; 785 unixFile *pNew; 786 int rc = SQLITE_OK; 787 int openFlags = O_RDWR | O_CREAT; 788 @@ -5815,7 +5812,7 @@ static int proxyCreateUnixFile( 789 pUnused->flags = openFlags; 790 pNew->pUnused = pUnused; 791 792 - rc = fillInUnixFile(&dummyVfs, fd, dirfd, (sqlite3_file*)pNew, path, 0, 0, 0); 793 + rc = fillInUnixFile(&dummyVfs, fd, 0, (sqlite3_file*)pNew, path, 0, 0, 0); 794 if( rc==SQLITE_OK ){ 795 *ppFile = pNew; 796 return SQLITE_OK; 797 @@ -5929,7 +5926,7 @@ static int proxyBreakConchLock(unixFile *pFile, uuid_t myHostID){ 798 end_breaklock: 799 if( rc ){ 800 if( fd>=0 ){ 801 - unlink(tPath); 802 + osUnlink(tPath); 803 robust_close(pFile, fd, __LINE__); 804 } 805 fprintf(stderr, "failed to break stale lock on %s, %s\n", cPath, errmsg); 806 @@ -6752,7 +6749,7 @@ int sqlite3_os_init(void){ 807 808 /* Double-check that the aSyscall[] array has been constructed 809 ** correctly. See ticket [bb3a86e890c8e96ab] */ 810 - assert( ArraySize(aSyscall)==16 ); 811 + assert( ArraySize(aSyscall)==18 ); 812 813 /* Register all VFSes defined in the aVfs[] array */ 814 for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ 815 diff --git a/third_party/sqlite/src/test/syscall.test b/third_party/sqlite/src/test/syscall.test 816 index 4442612..201bd63 100644 817 --- a/third_party/sqlite/src/test/syscall.test 818 +++ b/third_party/sqlite/src/test/syscall.test 819 @@ -59,7 +59,7 @@ do_test 2.1.2 { test_syscall exists nosuchcall } 0 820 foreach s { 821 open close access getcwd stat fstat ftruncate 822 fcntl read pread write pwrite fchmod fallocate 823 - pread64 pwrite64 824 + pread64 pwrite64 unlink openDirectory 825 } { 826 if {[test_syscall exists $s]} {lappend syscall_list $s} 827 } 828 diff --git a/third_party/sqlite/system-sqlite.patch b/third_party/sqlite/system-sqlite.patch 829 index f61f019..31d6b00 100644 830 --- a/third_party/sqlite/system-sqlite.patch 831 +++ b/third_party/sqlite/system-sqlite.patch 832 @@ -1,48 +0,0 @@ 833 -This is a backport of http://sqlite.org/src/ci/9109128cb5, 834 -which is needed for experiments with using unpatched sqlite. 835 -If you hit a merge conflict on this file it is most likely 836 -that you've upgraded to version of sqlite that includes this patch. 837 -Index: src/os_unix.c 838 -=================================================================== 839 ---- src/os_unix.c 840 -+++ src/os_unix.c 841 -@@ -4787,11 +4787,11 @@ 842 - ** ignored and -1 is returned. The caller will try to open a new file 843 - ** descriptor on the same path, fail, and return an error to SQLite. 844 - ** 845 - ** Even if a subsequent open() call does succeed, the consequences of 846 - ** not searching for a resusable file descriptor are not dire. */ 847 -- if( 0==stat(zPath, &sStat) ){ 848 -+ if( 0==osStat(zPath, &sStat) ){ 849 - unixInodeInfo *pInode; 850 - 851 - unixEnterMutex(); 852 - pInode = inodeList; 853 - while( pInode && (pInode->fileId.dev!=sStat.st_dev 854 -@@ -4863,11 +4863,11 @@ 855 - while( nDb>0 && zPath[nDb]!='-' ) nDb--; 856 - if( nDb==0 ) return SQLITE_OK; 857 - memcpy(zDb, zPath, nDb); 858 - zDb[nDb] = '\0'; 859 - 860 -- if( 0==stat(zDb, &sStat) ){ 861 -+ if( 0==osStat(zDb, &sStat) ){ 862 - *pMode = sStat.st_mode & 0777; 863 - }else{ 864 - rc = SQLITE_IOERR_FSTAT; 865 - } 866 - }else if( flags & SQLITE_OPEN_DELETEONCLOSE ){ 867 -@@ -5208,11 +5208,11 @@ 868 - assert(!"Invalid flags argument"); 869 - } 870 - *pResOut = (osAccess(zPath, amode)==0); 871 - if( flags==SQLITE_ACCESS_EXISTS && *pResOut ){ 872 - struct stat buf; 873 -- if( 0==stat(zPath, &buf) && buf.st_size==0 ){ 874 -+ if( 0==osStat(zPath, &buf) && buf.st_size==0 ){ 875 - *pResOut = 0; 876 - } 877 - } 878 - return SQLITE_OK; 879 - } 880 - 881
