diff options
Diffstat (limited to 'tool')
91 files changed, 30870 insertions, 0 deletions
diff --git a/tool/GetFile.cs b/tool/GetFile.cs new file mode 100644 index 0000000..1784a79 --- /dev/null +++ b/tool/GetFile.cs @@ -0,0 +1,471 @@ +/* +** 2015 October 7 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains C# code to download a single file based on a URI. +*/ +
+using System;
+using System.ComponentModel;
+using System.Diagnostics;
+using System.IO;
+using System.Net;
+using System.Reflection;
+using System.Runtime.InteropServices;
+using System.Threading;
+
+///////////////////////////////////////////////////////////////////////////////
+
+#region Assembly Metadata
+[assembly: AssemblyTitle("GetFile Tool")]
+[assembly: AssemblyDescription("Download a single file based on a URI.")]
+[assembly: AssemblyCompany("SQLite Development Team")]
+[assembly: AssemblyProduct("SQLite")]
+[assembly: AssemblyCopyright("Public Domain")]
+[assembly: ComVisible(false)]
+[assembly: Guid("5c4b3728-1693-4a33-a218-8e6973ca15a6")]
+[assembly: AssemblyVersion("1.0.*")]
+
+#if DEBUG
+[assembly: AssemblyConfiguration("Debug")]
+#else
+[assembly: AssemblyConfiguration("Release")]
+#endif
+#endregion
+
+///////////////////////////////////////////////////////////////////////////////
+
+namespace GetFile
+{
+ /// <summary>
+ /// This enumeration is used to represent all the possible exit codes from
+ /// this tool.
+ /// </summary>
+ internal enum ExitCode
+ {
+ /// <summary>
+ /// The file download was a success.
+ /// </summary>
+ Success = 0,
+
+ /// <summary>
+ /// The command line arguments are missing (i.e. null). Generally,
+ /// this should not happen.
+ /// </summary>
+ MissingArgs = 1,
+
+ /// <summary>
+ /// The wrong number of command line arguments was supplied.
+ /// </summary>
+ WrongNumArgs = 2,
+
+ /// <summary>
+ /// The URI specified on the command line could not be parsed as a
+ /// supported absolute URI.
+ /// </summary>
+ BadUri = 3,
+
+ /// <summary>
+ /// The file name portion of the URI specified on the command line
+ /// could not be extracted from it.
+ /// </summary>
+ BadFileName = 4,
+
+ /// <summary>
+ /// The temporary directory is either invalid (i.e. null) or does not
+ /// represent an available directory.
+ /// </summary>
+ BadTempPath = 5,
+
+ /// <summary>
+ /// An exception was caught in <see cref="Main" />. Generally, this
+ /// should not happen.
+ /// </summary>
+ Exception = 6,
+
+ /// <summary>
+ /// The file download was canceled. This tool does not make use of
+ /// the <see cref="WebClient.CancelAsync" /> method; therefore, this
+ /// should not happen.
+ /// </summary>
+ DownloadCanceled = 7,
+
+ /// <summary>
+ /// The file download encountered an error. Further information about
+ /// this error should be displayed on the console.
+ /// </summary>
+ DownloadError = 8
+ }
+
+ ///////////////////////////////////////////////////////////////////////////
+
+ internal static class Program
+ {
+ #region Private Data
+ /// <summary>
+ /// This is used to synchronize multithreaded access to the
+ /// <see cref="previousPercent" /> and <see cref="exitCode"/>
+ /// fields.
+ /// </summary>
+ private static readonly object syncRoot = new object();
+
+ ///////////////////////////////////////////////////////////////////////
+
+ /// <summary>
+ /// This event will be signed when the file download has completed,
+ /// even if the file download itself was canceled or unsuccessful.
+ /// </summary>
+ private static EventWaitHandle doneEvent;
+
+ ///////////////////////////////////////////////////////////////////////
+
+ /// <summary>
+ /// The previous file download completion percentage seen by the
+ /// <see cref="DownloadProgressChanged" /> event handler. This value
+ /// is never decreased, nor is it ever reset to zero.
+ /// </summary>
+ private static int previousPercent = 0;
+
+ ///////////////////////////////////////////////////////////////////////
+
+ /// <summary>
+ /// This will be the exit code returned by this tool after the file
+ /// download completes, successfully or otherwise. This value is only
+ /// changed by the <see cref="DownloadFileCompleted" /> event handler.
+ /// </summary>
+ private static ExitCode exitCode = ExitCode.Success;
+ #endregion
+
+ ///////////////////////////////////////////////////////////////////////
+
+ #region Private Support Methods
+ /// <summary>
+ /// This method displays an error message to the console and/or
+ /// displays the command line usage information for this tool.
+ /// </summary>
+ /// <param name="message">
+ /// The error message to display, if any.
+ /// </param>
+ /// <param name="usage">
+ /// Non-zero to display the command line usage information.
+ /// </param>
+ private static void Error(
+ string message,
+ bool usage
+ )
+ {
+ if (message != null)
+ Console.WriteLine(message);
+
+ string fileName = Path.GetFileName(
+ Process.GetCurrentProcess().MainModule.FileName);
+
+ Console.WriteLine(String.Format(
+ "usage: {0} <uri> [fileName]", fileName));
+ }
+
+ ///////////////////////////////////////////////////////////////////////
+
+ /// <summary>
+ /// This method attempts to determine the file name portion of the
+ /// specified URI.
+ /// </summary>
+ /// <param name="uri">
+ /// The URI to process.
+ /// </param>
+ /// <returns>
+ /// The file name portion of the specified URI -OR- null if it cannot
+ /// be determined.
+ /// </returns>
+ private static string GetFileName(
+ Uri uri
+ )
+ {
+ if (uri == null)
+ return null;
+
+ string pathAndQuery = uri.PathAndQuery;
+
+ if (String.IsNullOrEmpty(pathAndQuery))
+ return null;
+
+ int index = pathAndQuery.LastIndexOf('/');
+
+ if ((index < 0) || (index == pathAndQuery.Length))
+ return null;
+
+ return pathAndQuery.Substring(index + 1);
+ }
+ #endregion
+
+ ///////////////////////////////////////////////////////////////////////
+
+ #region Private Event Handlers
+ /// <summary>
+ /// This method is an event handler that is called when the file
+ /// download completion percentage changes. It will display progress
+ /// on the console. Special care is taken to make sure that progress
+ /// events are not displayed out-of-order, even if duplicate and/or
+ /// out-of-order events are received.
+ /// </summary>
+ /// <param name="sender">
+ /// The source of the event.
+ /// </param>
+ /// <param name="e">
+ /// Information for the event being processed.
+ /// </param>
+ private static void DownloadProgressChanged(
+ object sender,
+ DownloadProgressChangedEventArgs e
+ )
+ {
+ if (e != null)
+ {
+ int percent = e.ProgressPercentage;
+
+ lock (syncRoot)
+ {
+ if (percent > previousPercent)
+ {
+ Console.Write('.');
+
+ if ((percent % 10) == 0)
+ Console.Write(" {0}% ", percent);
+
+ previousPercent = percent;
+ }
+ }
+ }
+ }
+
+ ///////////////////////////////////////////////////////////////////////
+
+ /// <summary>
+ /// This method is an event handler that is called when the file
+ /// download has completed, successfully or otherwise. It will
+ /// display the overall result of the file download on the console,
+ /// including any <see cref="Exception" /> information, if applicable.
+ /// The <see cref="exitCode" /> field is changed by this method to
+ /// indicate the overall result of the file download and the event
+ /// within the <see cref="doneEvent" /> field will be signaled.
+ /// </summary>
+ /// <param name="sender">
+ /// The source of the event.
+ /// </param>
+ /// <param name="e">
+ /// Information for the event being processed.
+ /// </param>
+ private static void DownloadFileCompleted(
+ object sender,
+ AsyncCompletedEventArgs e
+ )
+ {
+ if (e != null)
+ {
+ lock (syncRoot)
+ {
+ if (previousPercent < 100)
+ Console.Write(' ');
+ }
+
+ if (e.Cancelled)
+ {
+ Console.WriteLine("Canceled");
+
+ lock (syncRoot)
+ {
+ exitCode = ExitCode.DownloadCanceled;
+ }
+ }
+ else
+ {
+ Exception error = e.Error;
+
+ if (error != null)
+ {
+ Console.WriteLine("Error: {0}", error);
+
+ lock (syncRoot)
+ {
+ exitCode = ExitCode.DownloadError;
+ }
+ }
+ else
+ {
+ Console.WriteLine("Done");
+ }
+ }
+ }
+
+ if (doneEvent != null)
+ doneEvent.Set();
+ }
+ #endregion
+
+ ///////////////////////////////////////////////////////////////////////
+
+ #region Program Entry Point
+ /// <summary>
+ /// This is the entry-point for this tool. It handles processing the
+ /// command line arguments, setting up the web client, downloading the
+ /// file, and saving it to the file system.
+ /// </summary>
+ /// <param name="args">
+ /// The command line arguments.
+ /// </param>
+ /// <returns>
+ /// Zero upon success; non-zero on failure. This will be one of the
+ /// values from the <see cref="ExitCode" /> enumeration.
+ /// </returns>
+ private static int Main(
+ string[] args
+ )
+ {
+ //
+ // NOTE: Sanity check the command line arguments.
+ //
+ if (args == null)
+ {
+ Error(null, true);
+ return (int)ExitCode.MissingArgs;
+ }
+
+ if ((args.Length < 1) || (args.Length > 2))
+ {
+ Error(null, true);
+ return (int)ExitCode.WrongNumArgs;
+ }
+
+ //
+ // NOTE: Attempt to convert the first (and only) command line
+ // argument to an absolute URI.
+ //
+ Uri uri;
+
+ if (!Uri.TryCreate(args[0], UriKind.Absolute, out uri))
+ {
+ Error("Could not create absolute URI from argument.", false);
+ return (int)ExitCode.BadUri;
+ }
+
+ //
+ // NOTE: If a file name was specified on the command line, try to
+ // use it (without its directory name); otherwise, fallback
+ // to using the file name portion of the URI.
+ //
+ string fileName = (args.Length == 2) ?
+ Path.GetFileName(args[1]) : null;
+
+ if (String.IsNullOrEmpty(fileName))
+ {
+ //
+ // NOTE: Attempt to extract the file name portion of the URI
+ // we just created.
+ //
+ fileName = GetFileName(uri);
+
+ if (fileName == null)
+ {
+ Error("Could not extract file name from URI.", false);
+ return (int)ExitCode.BadFileName;
+ }
+ }
+
+ //
+ // NOTE: Grab the temporary path setup for this process. If it is
+ // unavailable, we will not continue.
+ //
+ string directory = Path.GetTempPath();
+
+ if (String.IsNullOrEmpty(directory) ||
+ !Directory.Exists(directory))
+ {
+ Error("Temporary directory is invalid or unavailable.", false);
+ return (int)ExitCode.BadTempPath;
+ }
+
+ try
+ {
+ //
+ // HACK: For use of the TLS 1.2 security protocol because some
+ // web servers fail without it. In order to support the
+ // .NET Framework 2.0+ at compilation time, must use its
+ // integer constant here.
+ //
+ ServicePointManager.SecurityProtocol =
+ (SecurityProtocolType)0xC00;
+
+ using (WebClient webClient = new WebClient())
+ {
+ //
+ // NOTE: Create the event used to signal completion of the
+ // file download.
+ //
+ doneEvent = new ManualResetEvent(false);
+
+ //
+ // NOTE: Hookup the event handlers we care about on the web
+ // client. These are necessary because the file is
+ // downloaded asynchronously.
+ //
+ webClient.DownloadProgressChanged +=
+ new DownloadProgressChangedEventHandler(
+ DownloadProgressChanged);
+
+ webClient.DownloadFileCompleted +=
+ new AsyncCompletedEventHandler(
+ DownloadFileCompleted);
+
+ //
+ // NOTE: Build the fully qualified path and file name,
+ // within the temporary directory, where the file to
+ // be downloaded will be saved.
+ //
+ fileName = Path.Combine(directory, fileName);
+
+ //
+ // NOTE: If the file name already exists (in the temporary)
+ // directory, delete it.
+ //
+ // TODO: Perhaps an error should be raised here instead?
+ //
+ if (File.Exists(fileName))
+ File.Delete(fileName);
+
+ //
+ // NOTE: After kicking off the asynchronous file download
+ // process, wait [forever] until the "done" event is
+ // signaled.
+ //
+ Console.WriteLine(
+ "Downloading \"{0}\" to \"{1}\"...", uri, fileName);
+
+ webClient.DownloadFileAsync(uri, fileName);
+ doneEvent.WaitOne();
+ }
+
+ lock (syncRoot)
+ {
+ return (int)exitCode;
+ }
+ }
+ catch (Exception e)
+ {
+ //
+ // NOTE: An exception was caught. Report it via the console
+ // and return failure.
+ //
+ Error(e.ToString(), false);
+ return (int)ExitCode.Exception;
+ }
+ }
+ #endregion
+ }
+}
diff --git a/tool/GetTclKit.bat b/tool/GetTclKit.bat new file mode 100644 index 0000000..cd295a0 --- /dev/null +++ b/tool/GetTclKit.bat @@ -0,0 +1,333 @@ +@ECHO OFF
+
+::
+:: GetTclKit.bat --
+::
+:: TclKit Download Tool
+::
+
+SETLOCAL
+
+REM SET __ECHO=ECHO
+REM SET __ECHO2=ECHO
+REM SET __ECHO3=ECHO
+IF NOT DEFINED _AECHO (SET _AECHO=REM)
+IF NOT DEFINED _CECHO (SET _CECHO=REM)
+IF NOT DEFINED _CECHO2 (SET _CECHO2=REM)
+IF NOT DEFINED _CECHO3 (SET _CECHO3=REM)
+IF NOT DEFINED _VECHO (SET _VECHO=REM)
+
+SET OVERWRITE=^>
+IF DEFINED __ECHO SET OVERWRITE=^^^>
+
+SET APPEND=^>^>
+IF DEFINED __ECHO SET APPEND=^^^>^^^>
+
+SET PROCESSOR=%1
+
+IF DEFINED PROCESSOR (
+ CALL :fn_UnquoteVariable PROCESSOR
+) ELSE (
+ GOTO usage
+)
+
+SET PROCESSOR=%PROCESSOR:AMD64=x64%
+
+%_VECHO% Processor = '%PROCESSOR%'
+
+SET DUMMY2=%2
+
+IF DEFINED DUMMY2 (
+ GOTO usage
+)
+
+IF NOT DEFINED ENVDIR (
+ SET ENVDIR=%CD%
+)
+
+%_VECHO% EnvDir = '%ENVDIR%'
+
+SET TOOLS=%~dp0
+SET TOOLS=%TOOLS:~0,-1%
+
+%_VECHO% Tools = '%TOOLS%'
+
+IF NOT DEFINED windir (
+ ECHO The windir environment variable must be set first.
+ GOTO errors
+)
+
+%_VECHO% WinDir = '%windir%'
+
+IF NOT DEFINED TEMP (
+ ECHO The TEMP environment variable must be set first.
+ GOTO errors
+)
+
+%_VECHO% Temp = '%TEMP%'
+
+IF NOT DEFINED TCLKIT_URI (
+ SET TCLKIT_URI=https://urn.to/r/tclsh/
+)
+
+%_VECHO% TclKitUri = '%TCLKIT_URI%'
+
+IF NOT DEFINED TCLKIT_PATCHLEVEL (
+ SET TCLKIT_PATCHLEVEL=8.6.6
+)
+
+%_VECHO% TclKitPatchLevel = '%TCLKIT_PATCHLEVEL%'
+
+IF NOT DEFINED TCLKIT_EXE_PATCHLEVEL (
+ SET TCLKIT_EXE_PATCHLEVEL=8.6.4
+)
+
+%_VECHO% TclKitExePatchLevel = '%TCLKIT_EXE_PATCHLEVEL%'
+
+IF /I "%PROCESSOR%" == "x86" (
+ CALL :fn_TclKitX86Variables
+
+ IF ERRORLEVEL 1 (
+ GOTO errors
+ )
+) ELSE IF /I "%PROCESSOR%" == "x64" (
+ CALL :fn_TclKitX64Variables
+
+ IF ERRORLEVEL 1 (
+ GOTO errors
+ )
+) ELSE (
+ GOTO usage
+)
+
+%_VECHO% TclKitVersion = '%TCLKIT_VERSION%'
+%_VECHO% TclKitPatchLevel = '%TCLKIT_PATCHLEVEL%'
+%_VECHO% TclKitExePatchLevel = '%TCLKIT_EXE_PATCHLEVEL%'
+%_VECHO% TclKitNoEnv = '%TCLKIT_NOENV%'
+%_VECHO% TclKitNoSdk = '%TCLKIT_NOSDK%'
+%_VECHO% TclKitExe = '%TCLKIT_EXE%'
+%_VECHO% TclKitLib = '%TCLKIT_LIB%'
+%_VECHO% TclKitLibStub = '%TCLKIT_LIB_STUB%'
+%_VECHO% TclKitSdk = '%TCLKIT_SDK%'
+%_VECHO% TclKitSdkZip = '%TCLKIT_SDK_ZIP%'
+%_VECHO% TclKitFiles = '%TCLKIT_FILES%'
+
+CALL :fn_ResetErrorLevel
+
+FOR %%T IN (csc.exe) DO (
+ SET %%T_PATH=%%~dp$PATH:T
+)
+
+%_VECHO% Csc.exe_PATH = '%csc.exe_PATH%'
+
+IF DEFINED csc.exe_PATH (
+ GOTO skip_addToPath
+)
+
+IF DEFINED FRAMEWORKDIR (
+ REM Use the existing .NET Framework directory...
+) ELSE IF EXIST "%windir%\Microsoft.NET\Framework64\v2.0.50727" (
+ SET FRAMEWORKDIR=%windir%\Microsoft.NET\Framework64\v2.0.50727
+) ELSE IF EXIST "%windir%\Microsoft.NET\Framework64\v3.5" (
+ SET FRAMEWORKDIR=%windir%\Microsoft.NET\Framework64\v3.5
+) ELSE IF EXIST "%windir%\Microsoft.NET\Framework64\v4.0.30319" (
+ SET FRAMEWORKDIR=%windir%\Microsoft.NET\Framework64\v4.0.30319
+) ELSE IF EXIST "%windir%\Microsoft.NET\Framework\v2.0.50727" (
+ SET FRAMEWORKDIR=%windir%\Microsoft.NET\Framework\v2.0.50727
+) ELSE IF EXIST "%windir%\Microsoft.NET\Framework\v3.5" (
+ SET FRAMEWORKDIR=%windir%\Microsoft.NET\Framework\v3.5
+) ELSE IF EXIST "%windir%\Microsoft.NET\Framework\v4.0.30319" (
+ SET FRAMEWORKDIR=%windir%\Microsoft.NET\Framework\v4.0.30319
+) ELSE (
+ ECHO No suitable version of the .NET Framework appears to be installed.
+ GOTO errors
+)
+
+%_VECHO% FrameworkDir = '%FRAMEWORKDIR%'
+
+IF NOT EXIST "%FRAMEWORKDIR%\csc.exe" (
+ ECHO The file "%FRAMEWORKDIR%\csc.exe" is missing.
+ GOTO errors
+)
+
+CALL :fn_PrependToPath FRAMEWORKDIR
+
+:skip_addToPath
+
+IF NOT EXIST "%TEMP%\GetFile.exe" (
+ %__ECHO% csc.exe "/out:%TEMP%\GetFile.exe" /target:exe "%TOOLS%\GetFile.cs"
+
+ IF ERRORLEVEL 1 (
+ ECHO Compilation of "%TOOLS%\GetFile.cs" failed.
+ GOTO errors
+ )
+)
+
+FOR %%F IN (%TCLKIT_FILES%) DO (
+ IF NOT EXIST "%TEMP%\%%F" (
+ %__ECHO% "%TEMP%\GetFile.exe" "%TCLKIT_URI%%%F"
+
+ IF ERRORLEVEL 1 (
+ ECHO Download of "%%F" from "%TCLKIT_URI%" failed.
+ GOTO errors
+ )
+ )
+)
+
+IF DEFINED TCLKIT_NOENV GOTO skip_sdkUnZip
+IF DEFINED TCLKIT_NOSDK GOTO skip_sdkUnZip
+
+IF NOT EXIST "%TEMP%\%TCLKIT_SDK%" (
+ %__ECHO% MKDIR "%TEMP%\%TCLKIT_SDK%"
+
+ IF ERRORLEVEL 1 (
+ ECHO Could not create directory "%TEMP%\%TCLKIT_SDK%".
+ GOTO errors
+ )
+)
+
+%__ECHO% "%TEMP%\unzip.exe" -n "%TEMP%\%TCLKIT_SDK_ZIP%" -d "%TEMP%\%TCLKIT_SDK%"
+
+IF ERRORLEVEL 1 (
+ ECHO Could not unzip "%TEMP%\%TCLKIT_SDK_ZIP%" to "%TEMP%\%TCLKIT_SDK%".
+ GOTO errors
+)
+
+:skip_sdkUnZip
+
+IF DEFINED TCLKIT_NOENV GOTO skip_sdkEnvironment
+
+%__ECHO% ECHO SET TCLSH_CMD=%TEMP%\%TCLKIT_EXE%%OVERWRITE%"%ENVDIR%\SetTclKitEnv.bat"
+
+IF DEFINED TCLKIT_NOSDK GOTO skip_sdkVariables
+
+%__ECHO% ECHO SET TCLINCDIR=%TEMP%\%TCLKIT_SDK%\include%APPEND%"%ENVDIR%\SetTclKitEnv.bat"
+%__ECHO% ECHO SET TCLLIBDIR=%TEMP%\%TCLKIT_SDK%\lib%APPEND%"%ENVDIR%\SetTclKitEnv.bat"
+%__ECHO% ECHO SET LIBTCLPATH=%TEMP%\%TCLKIT_SDK%\lib%APPEND%"%ENVDIR%\SetTclKitEnv.bat"
+%__ECHO% ECHO SET LIBTCL=%TCLKIT_LIB%%APPEND%"%ENVDIR%\SetTclKitEnv.bat"
+%__ECHO% ECHO SET LIBTCLSTUB=%TCLKIT_LIB_STUB%%APPEND%"%ENVDIR%\SetTclKitEnv.bat"
+
+:skip_sdkVariables
+
+ECHO.
+ECHO Wrote "%ENVDIR%\SetTclKitEnv.bat".
+ECHO Please run it to set the necessary Tcl environment variables.
+ECHO.
+
+:skip_sdkEnvironment
+
+GOTO no_errors
+
+:fn_TclKitX86Variables
+ REM
+ REM NOTE: By default, use latest available version of the TclKit SDK
+ REM for x86. However, the "default" TclKit executable for x86
+ REM is still used here because it is the only one "well-known"
+ REM to be available for download.
+ REM
+ IF NOT DEFINED TCLKIT_PATCHLEVEL (
+ ECHO The TCLKIT_PATCHLEVEL environment variable must be set first.
+ CALL :fn_SetErrorLevel
+ GOTO :EOF
+ )
+ SET TCLKIT_VERSION=%TCLKIT_PATCHLEVEL:.=%
+ SET TCLKIT_VERSION=%TCLKIT_VERSION:~0,2%
+ IF DEFINED TCLKIT_EXE_PATCHLEVEL (
+ SET TCLKIT_EXE=tclkit-%TCLKIT_EXE_PATCHLEVEL%.exe
+ ) ELSE (
+ SET TCLKIT_EXE=tclkit-%TCLKIT_PATCHLEVEL%.exe
+ )
+ SET TCLKIT_LIB=libtclkit%TCLKIT_PATCHLEVEL:.=%.lib
+ SET TCLKIT_LIB_STUB=libtclstub%TCLKIT_VERSION:.=%.a
+ SET TCLKIT_SDK=libtclkit-sdk-x86-%TCLKIT_PATCHLEVEL%
+ SET TCLKIT_SDK_ZIP=%TCLKIT_SDK%.zip
+ SET TCLKIT_FILES=%TCLKIT_EXE%
+ IF NOT DEFINED TCLKIT_NOENV IF NOT DEFINED TCLKIT_NOSDK (
+ SET TCLKIT_FILES=%TCLKIT_FILES% unzip.exe %TCLKIT_SDK_ZIP%
+ )
+ GOTO :EOF
+
+:fn_TclKitX64Variables
+ REM
+ REM NOTE: By default, use latest available version of the TclKit SDK
+ REM for x64. However, the "default" TclKit executable for x86
+ REM is still used here because it is the only one "well-known"
+ REM to be available for download.
+ REM
+ IF NOT DEFINED TCLKIT_PATCHLEVEL (
+ ECHO The TCLKIT_PATCHLEVEL environment variable must be set first.
+ CALL :fn_SetErrorLevel
+ GOTO :EOF
+ )
+ SET TCLKIT_VERSION=%TCLKIT_PATCHLEVEL:.=%
+ SET TCLKIT_VERSION=%TCLKIT_VERSION:~0,2%
+ IF DEFINED TCLKIT_EXE_PATCHLEVEL (
+ SET TCLKIT_EXE=tclkit-%TCLKIT_EXE_PATCHLEVEL%.exe
+ ) ELSE (
+ SET TCLKIT_EXE=tclkit-%TCLKIT_PATCHLEVEL%.exe
+ )
+ SET TCLKIT_LIB=libtclkit%TCLKIT_PATCHLEVEL:.=%.lib
+ SET TCLKIT_LIB_STUB=libtclstub%TCLKIT_VERSION:.=%.a
+ SET TCLKIT_SDK=libtclkit-sdk-x64-%TCLKIT_PATCHLEVEL%
+ SET TCLKIT_SDK_ZIP=%TCLKIT_SDK%.zip
+ SET TCLKIT_FILES=%TCLKIT_EXE%
+ IF NOT DEFINED TCLKIT_NOENV IF NOT DEFINED TCLKIT_NOSDK (
+ SET TCLKIT_FILES=%TCLKIT_FILES% unzip.exe %TCLKIT_SDK_ZIP%
+ )
+ GOTO :EOF
+
+:fn_UnquoteVariable
+ IF NOT DEFINED %1 GOTO :EOF
+ SETLOCAL
+ SET __ECHO_CMD=ECHO %%%1%%
+ FOR /F "delims=" %%V IN ('%__ECHO_CMD%') DO (
+ SET VALUE=%%V
+ )
+ SET VALUE=%VALUE:"=%
+ REM "
+ ENDLOCAL && SET %1=%VALUE%
+ GOTO :EOF
+
+:fn_PrependToPath
+ IF NOT DEFINED %1 GOTO :EOF
+ SETLOCAL
+ SET __ECHO_CMD=ECHO %%%1%%
+ FOR /F "delims=" %%V IN ('%__ECHO_CMD%') DO (
+ SET VALUE=%%V
+ )
+ SET VALUE=%VALUE:"=%
+ REM "
+ ENDLOCAL && SET PATH=%VALUE%;%PATH%
+ GOTO :EOF
+
+:fn_ResetErrorLevel
+ VERIFY > NUL
+ GOTO :EOF
+
+:fn_SetErrorLevel
+ VERIFY MAYBE 2> NUL
+ GOTO :EOF
+
+:usage
+ ECHO.
+ ECHO Usage: %~nx0 ^<processor^>
+ ECHO.
+ ECHO The only supported values for processor are "x86" and "x64".
+ GOTO errors
+
+:errors
+ CALL :fn_SetErrorLevel
+ ENDLOCAL
+ ECHO.
+ ECHO Failure, errors were encountered.
+ GOTO end_of_file
+
+:no_errors
+ CALL :fn_ResetErrorLevel
+ ENDLOCAL
+ ECHO.
+ ECHO Success, no errors were encountered.
+ GOTO end_of_file
+
+:end_of_file
+%__ECHO% EXIT /B %ERRORLEVEL%
diff --git a/tool/Replace.cs b/tool/Replace.cs new file mode 100644 index 0000000..3475a47 --- /dev/null +++ b/tool/Replace.cs @@ -0,0 +1,223 @@ +/* +** 2016 February 26 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains C# code to perform regular expression replacements +** using the standard input and output channels. +*/ + +using System; +using System.Diagnostics; +using System.IO; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Text.RegularExpressions; + +/////////////////////////////////////////////////////////////////////////////// + +#region Assembly Metadata +[assembly: AssemblyTitle("Replace Tool")] +[assembly: AssemblyDescription("Replace text using standard input/output.")] +[assembly: AssemblyCompany("SQLite Development Team")] +[assembly: AssemblyProduct("SQLite")] +[assembly: AssemblyCopyright("Public Domain")] +[assembly: ComVisible(false)] +[assembly: Guid("95a0513f-8863-48cd-a76f-cb80868cb578")] +[assembly: AssemblyVersion("1.0.*")] + +#if DEBUG +[assembly: AssemblyConfiguration("Debug")] +#else +[assembly: AssemblyConfiguration("Release")] +#endif +#endregion + +/////////////////////////////////////////////////////////////////////////////// + +namespace Replace +{ + /// <summary> + /// This enumeration is used to represent all the possible exit codes from + /// this tool. + /// </summary> + internal enum ExitCode + { + /// <summary> + /// The file download was a success. + /// </summary> + Success = 0, + + /// <summary> + /// The command line arguments are missing (i.e. null). Generally, + /// this should not happen. + /// </summary> + MissingArgs = 1, + + /// <summary> + /// The wrong number of command line arguments was supplied. + /// </summary> + WrongNumArgs = 2, + + /// <summary> + /// The "matchingOnly" flag could not be converted to a value of the + /// <see cref="Boolean"/> type. + /// </summary> + BadMatchingOnlyFlag = 3, + + /// <summary> + /// An exception was caught in <see cref="Main" />. Generally, this + /// should not happen. + /// </summary> + Exception = 4 + } + + /////////////////////////////////////////////////////////////////////////// + + internal static class Replace + { + #region Private Support Methods + /// <summary> + /// This method displays an error message to the console and/or + /// displays the command line usage information for this tool. + /// </summary> + /// <param name="message"> + /// The error message to display, if any. + /// </param> + /// <param name="usage"> + /// Non-zero to display the command line usage information. + /// </param> + private static void Error( + string message, + bool usage + ) + { + if (message != null) + Console.WriteLine(message); + + string fileName = Path.GetFileName( + Process.GetCurrentProcess().MainModule.FileName); + + Console.WriteLine(String.Format( + "usage: {0} <regExPattern> <regExSubSpec> <matchingOnly>", + fileName)); + } + #endregion + + /////////////////////////////////////////////////////////////////////// + + #region Program Entry Point + /// <summary> + /// This is the entry-point for this tool. It handles processing the + /// command line arguments, reading from the standard input channel, + /// replacing any matching lines of text, and writing to the standard + /// output channel. + /// </summary> + /// <param name="args"> + /// The command line arguments. + /// </param> + /// <returns> + /// Zero upon success; non-zero on failure. This will be one of the + /// values from the <see cref="ExitCode" /> enumeration. + /// </returns> + private static int Main( + string[] args + ) + { + // + // NOTE: Sanity check the command line arguments. + // + if (args == null) + { + Error(null, true); + return (int)ExitCode.MissingArgs; + } + + if (args.Length != 3) + { + Error(null, true); + return (int)ExitCode.WrongNumArgs; + } + + try + { + // + // NOTE: Create a regular expression from the first command + // line argument. Then, grab the replacement string, + // which is the second argument. + // + Regex regEx = new Regex(args[0]); + string replacement = args[1]; + + // + // NOTE: Attempt to convert the third argument to a boolean. + // + bool matchingOnly; + + if (!bool.TryParse(args[2], out matchingOnly)) + { + Error(null, true); + return (int)ExitCode.BadMatchingOnlyFlag; + } + + // + // NOTE: Grab the standard input and output channels from the + // console. + // + TextReader inputTextReader = Console.In; + TextWriter outputTextWriter = Console.Out; + + // + // NOTE: Loop until end-of-file is hit on the standard input + // stream. + // + while (true) + { + // + // NOTE: Read a line from the standard input channel. If + // null is returned here, there is no more input and + // we are done. + // + string inputLine = inputTextReader.ReadLine(); + + if (inputLine == null) + break; + + // + // NOTE: Perform regular expression replacements on this + // line, if any. Then, write the modified line to + // the standard output channel. + // + string outputLine = regEx.Replace(inputLine, replacement); + + if (!matchingOnly || !String.Equals( + inputLine, outputLine, StringComparison.Ordinal)) + { + outputTextWriter.WriteLine(outputLine); + } + } + + // + // NOTE: At this point, everything has succeeded. + // + return (int)ExitCode.Success; + } + catch (Exception e) + { + // + // NOTE: An exception was caught. Report it via the console + // and return failure. + // + Error(e.ToString(), false); + return (int)ExitCode.Exception; + } + } + #endregion + } +} diff --git a/tool/build-all-msvc.bat b/tool/build-all-msvc.bat new file mode 100755 index 0000000..8f9a1b7 --- /dev/null +++ b/tool/build-all-msvc.bat @@ -0,0 +1,864 @@ +@ECHO OFF
+
+::
+:: build-all-msvc.bat --
+::
+:: Multi-Platform Build Tool for MSVC
+::
+
+REM
+REM This batch script is used to build the SQLite DLL for multiple platforms
+REM and configurations using MSVC. The built SQLite DLLs, their associated
+REM import libraries, and optionally their symbols files, are placed within
+REM the directory specified on the command line, in sub-directories named for
+REM their respective platforms and configurations. This batch script must be
+REM run from inside a Visual Studio Command Prompt for the desired version of
+REM Visual Studio ^(the initial platform configured for the command prompt does
+REM not really matter^). Exactly one command line argument is required, the
+REM name of an existing directory to be used as the final destination directory
+REM for the generated output files, which will be placed in sub-directories
+REM created therein. Ideally, the directory specified should be empty.
+REM
+REM Example:
+REM
+REM CD /D C:\dev\sqlite\core
+REM CALL tool\build-all-msvc.bat C:\Temp
+REM
+REM In the example above, "C:\dev\sqlite\core" represents the root of the
+REM source tree for SQLite and "C:\Temp" represents the final destination
+REM directory for the generated output files.
+REM
+REM Please note that the SQLite build process performed by the Makefile
+REM associated with this batch script requires a Tcl shell to be present
+REM in a directory contained in the PATH environment variable unless a
+REM pre-existing amalgamation file is used.
+REM
+REM There are several environment variables that may be set to modify the
+REM behavior of this batch script and its associated Makefile. The list of
+REM platforms to build may be overriden by using the PLATFORMS environment
+REM variable, which should contain a list of platforms ^(e.g. x86 x86_amd64
+REM x86_arm^). All platforms must be supported by the version of Visual Studio
+REM being used. The list of configurations to build may be overridden by
+REM setting the CONFIGURATIONS environment variable, which should contain a
+REM list of configurations to build ^(e.g. Debug Retail^). Neither of these
+REM variable values may contain any double quotes, surrounding or embedded.
+REM
+REM Finally, the NCRTLIBPATH, NUCRTLIBPATH, and NSDKLIBPATH environment
+REM variables may be set to specify the location of the CRT, Universal CRT, and
+REM Windows SDK, respectively, that may be needed to compile executables native
+REM to the architecture of the build machine during any cross-compilation that
+REM may be necessary, depending on the platforms to be built. These values in
+REM these three variables should be surrounded by double quotes if they contain
+REM spaces.
+REM
+REM There are a few other environment variables that impact the build process
+REM when set ^(to anything^), they are:
+REM
+REM USE_AUTOCONF_MAKEFILE
+REM
+REM When set, the "autoconf" Makefile for MSVC will be used instead of the main
+REM Makefile for MSVC. It must exist at "%ROOT%\autoconf\Makefile.msc".
+REM
+REM NOCLEAN
+REM
+REM When set, the "clean" target will not be used during each build iteration.
+REM However, the target binaries, if any, will still be deleted manually prior
+REM to being rebuilt. Setting this environment variable is only rarely needed
+REM and could cause issues in some circumstances; therefore, setting it is not
+REM recommended.
+REM
+REM NOSYMBOLS
+REM
+REM When set, copying of symbol files ^(*.pdb^) created during the build will
+REM be skipped and they will not appear in the final destination directory.
+REM Setting this environment variable is never strictly needed and could cause
+REM issues in some circumstances; therefore, setting it is not recommended.
+REM
+REM NOMEMDEBUG
+REM
+REM When set, disables use of MEMDEBUG when building binaries for the "Debug"
+REM configuration.
+REM
+REM BUILD_ALL_SHELL
+REM
+REM When set, the command line shell will be built for each selected platform
+REM and configuration as well. In addition, the command line shell binaries
+REM will be copied, with their symbols, to the final destination directory.
+REM
+REM USE_WINV63_NSDKLIBPATH
+REM
+REM When set, modifies how the NSDKLIBPATH environment variable is built, based
+REM on the WindowsSdkDir environment variable. It forces this batch script to
+REM assume the Windows 8.1 SDK location should be used.
+REM
+REM USE_WINV100_NSDKLIBPATH
+REM
+REM When set, modifies how the NSDKLIBPATH environment variable is built, based
+REM on the WindowsSdkDir environment variable. It causes this batch script to
+REM assume the Windows 10.0 SDK location should be used.
+REM
+REM NMAKE_ARGS
+REM NMAKE_ARGS_DEBUG
+REM NMAKE_ARGS_RETAIL
+REM
+REM When set, these values are expanded and passed to the NMAKE command line,
+REM after its other arguments. These may be used to specify additional NMAKE
+REM options, for example:
+REM
+REM SET NMAKE_ARGS=FOR_WINRT=1
+REM SET NMAKE_ARGS_DEBUG=MEMDEBUG=1
+REM SET NMAKE_ARGS_RETAIL=WIN32HEAP=1
+REM
+REM Using the above command before running this tool will cause the compiled
+REM binaries to target the WinRT environment, which provides a subset of the
+REM Win32 API.
+REM
+REM DLL_FILE_NAME
+REM DLL_PDB_FILE_NAME
+REM LIB_FILE_NAME
+REM EXE_FILE_NAME
+REM EXE_PDB_FILE_NAME
+REM
+REM When set, these values will override the associated target file name used
+REM for the build.
+REM
+SETLOCAL
+
+REM SET __ECHO=ECHO
+REM SET __ECHO2=ECHO
+REM SET __ECHO3=ECHO
+IF NOT DEFINED _AECHO (SET _AECHO=REM)
+IF NOT DEFINED _CECHO (SET _CECHO=REM)
+IF NOT DEFINED _CECHO2 (SET _CECHO2=REM)
+IF NOT DEFINED _CECHO3 (SET _CECHO3=REM)
+IF NOT DEFINED _VECHO (SET _VECHO=REM)
+
+SET REDIRECT=^>
+IF DEFINED __ECHO SET REDIRECT=^^^>
+
+%_AECHO% Running %0 %*
+
+REM SET DFLAGS=/L
+
+%_VECHO% DFlags = '%DFLAGS%'
+
+SET FFLAGS=/V /F /G /H /I /R /Y /Z
+
+%_VECHO% FFlags = '%FFLAGS%'
+
+SET ROOT=%~dp0\..
+SET ROOT=%ROOT:\\=\%
+
+%_VECHO% Root = '%ROOT%'
+
+REM
+REM NOTE: The first and only argument to this batch file should be the output
+REM directory where the platform-specific binary directories should be
+REM created.
+REM
+SET BINARYDIRECTORY=%1
+
+IF NOT DEFINED BINARYDIRECTORY (
+ GOTO usage
+)
+
+%_VECHO% BinaryDirectory = '%BINARYDIRECTORY%'
+
+SET DUMMY=%2
+
+IF DEFINED DUMMY (
+ GOTO usage
+)
+
+REM
+REM NOTE: From this point, we need a clean error level. Reset it now.
+REM
+CALL :fn_ResetErrorLevel
+
+REM
+REM NOTE: Change the current directory to the root of the source tree, saving
+REM the current directory on the directory stack.
+REM
+%_CECHO2% PUSHD "%ROOT%"
+%__ECHO2% PUSHD "%ROOT%"
+
+IF ERRORLEVEL 1 (
+ ECHO Could not change directory to "%ROOT%".
+ GOTO errors
+)
+
+REM
+REM NOTE: This batch file requires the ComSpec environment variable to be set,
+REM typically to something like "C:\Windows\System32\cmd.exe".
+REM
+IF NOT DEFINED ComSpec (
+ ECHO The ComSpec environment variable must be defined.
+ GOTO errors
+)
+
+REM
+REM NOTE: This batch file requires the VcInstallDir environment variable to be
+REM set. Tyipcally, this means this batch file needs to be run from an
+REM MSVC command prompt.
+REM
+IF NOT DEFINED VCINSTALLDIR (
+ ECHO The VCINSTALLDIR environment variable must be defined.
+ GOTO errors
+)
+
+REM
+REM NOTE: If the list of platforms is not already set, use the default list.
+REM
+IF NOT DEFINED PLATFORMS (
+ SET PLATFORMS=x86 x86_amd64 x86_arm
+)
+
+%_VECHO% Platforms = '%PLATFORMS%'
+
+REM
+REM NOTE: If the list of configurations is not already set, use the default
+REM list.
+REM
+IF NOT DEFINED CONFIGURATIONS (
+ SET CONFIGURATIONS=Debug Retail
+)
+
+%_VECHO% Configurations = '%CONFIGURATIONS%'
+
+REM
+REM NOTE: If the command used to invoke NMAKE is not already set, use the
+REM default.
+REM
+IF NOT DEFINED NMAKE_CMD (
+ IF DEFINED USE_AUTOCONF_MAKEFILE (
+ SET NMAKE_CMD=nmake -B -f autoconf\Makefile.msc
+ ) ELSE (
+ SET NMAKE_CMD=nmake -B -f Makefile.msc
+ )
+)
+
+%_VECHO% NmakeCmd = '%NMAKE_CMD%'
+%_VECHO% NmakeArgs = '%NMAKE_ARGS%'
+%_VECHO% NmakeArgsDebug = '%NMAKE_ARGS_DEBUG%'
+%_VECHO% NmakeArgsRetail = '%NMAKE_ARGS_RETAIL%'
+
+REM
+REM NOTE: Setup environment variables to translate between the MSVC platform
+REM names and the names to be used for the platform-specific binary
+REM directories.
+REM
+SET amd64_NAME=x64
+SET arm_NAME=ARM
+SET x64_NAME=x64
+SET x86_NAME=x86
+SET x86_amd64_NAME=x64
+SET x86_arm_NAME=ARM
+SET x86_x64_NAME=x64
+
+%_VECHO% amd64_Name = '%amd64_NAME%'
+%_VECHO% arm_Name = '%arm_NAME%'
+%_VECHO% x64_Name = '%x64_NAME%'
+%_VECHO% x86_Name = '%x86_NAME%'
+%_VECHO% x86_amd64_Name = '%x86_amd64_NAME%'
+%_VECHO% x86_arm_Name = '%x86_arm_NAME%'
+%_VECHO% x86_x64_Name = '%x86_x64_NAME%'
+
+REM
+REM NOTE: Check for the external tools needed during the build process ^(i.e.
+REM those that do not get compiled as part of the build process itself^)
+REM along the PATH.
+REM
+IF DEFINED TCLSH_CMD (
+ SET TCLSH_FILE=%TCLSH_CMD%
+) ELSE (
+ SET TCLSH_FILE=tclsh.exe
+)
+
+FOR %%T IN (%TCLSH_FILE%) DO (
+ SET %%T_PATH=%%~dp$PATH:T
+)
+
+REM
+REM NOTE: A Tcl shell executable is required during the SQLite build process
+REM unless a pre-existing amalgamation file is used.
+REM
+IF NOT DEFINED %TCLSH_FILE%_PATH (
+ ECHO The Tcl shell executable "%TCLSH_FILE%" is required to be in the PATH.
+ GOTO errors
+)
+
+REM
+REM NOTE: Setup the default names for the build targets we are creating. Any
+REM ^(or all^) of these may end up being overridden.
+REM
+IF NOT DEFINED DLL_FILE_NAME (
+ SET DLL_FILE_NAME=sqlite3.dll
+)
+
+IF NOT DEFINED DLL_PDB_FILE_NAME (
+ SET DLL_PDB_FILE_NAME=sqlite3.pdb
+)
+
+IF NOT DEFINED LIB_FILE_NAME (
+ SET LIB_FILE_NAME=sqlite3.lib
+)
+
+IF NOT DEFINED EXE_FILE_NAME (
+ SET EXE_FILE_NAME=sqlite3.exe
+)
+
+IF NOT DEFINED EXE_PDB_FILE_NAME (
+ SET EXE_PDB_FILE_NAME=sqlite3sh.pdb
+)
+
+REM
+REM NOTE: Set the TOOLPATH variable to contain all the directories where the
+REM external tools were found in the search above.
+REM
+CALL :fn_CopyVariable %TCLSH_FILE%_PATH TOOLPATH
+
+%_VECHO% ToolPath = '%TOOLPATH%'
+
+REM
+REM NOTE: Setting the Windows SDK library path is only required for MSVC
+REM 2012, 2013, and 2015.
+REM
+CALL :fn_UnsetVariable SET_NSDKLIBPATH
+
+REM
+REM NOTE: Setting the Universal CRT library path is only required for MSVC
+REM 2015.
+REM
+CALL :fn_UnsetVariable SET_NUCRTLIBPATH
+
+REM
+REM NOTE: Check for MSVC 2012, 2013, and 2015 specially because the Windows
+REM SDK directory handling is slightly different for those versions.
+REM
+IF "%VisualStudioVersion%" == "11.0" (
+ REM
+ REM NOTE: If the Windows SDK library path has already been set, do not set
+ REM it to something else later on.
+ REM
+ IF NOT DEFINED NSDKLIBPATH (
+ SET SET_NSDKLIBPATH=1
+ )
+) ELSE IF "%VisualStudioVersion%" == "12.0" (
+ REM
+ REM NOTE: If the Windows SDK library path has already been set, do not set
+ REM it to something else later on.
+ REM
+ IF NOT DEFINED NSDKLIBPATH (
+ SET SET_NSDKLIBPATH=1
+ )
+) ELSE IF "%VisualStudioVersion%" == "14.0" (
+ REM
+ REM NOTE: If the Windows SDK library path has already been set, do not set
+ REM it to something else later on.
+ REM
+ IF NOT DEFINED NSDKLIBPATH (
+ SET SET_NSDKLIBPATH=1
+ )
+
+ REM
+ REM NOTE: If the Universal CRT library path has already been set, do not set
+ REM it to something else later on.
+ REM
+ IF NOT DEFINED NUCRTLIBPATH (
+ SET SET_NUCRTLIBPATH=1
+ )
+)
+
+REM
+REM NOTE: This is the name of the sub-directory where the UCRT libraries may
+REM be found. It is only used when compiling against the UCRT.
+REM
+IF DEFINED UCRTVersion (
+ SET NUCRTVER=%UCRTVersion%
+) ELSE (
+ SET NUCRTVER=10.0.10586.0
+)
+
+REM
+REM NOTE: This is the name of the sub-directory where the Windows 10.0 SDK
+REM libraries may be found. It is only used when compiling with the
+REM Windows 10.0 SDK.
+REM
+IF DEFINED WindowsSDKLibVersion (
+ SET WIN10SDKVER=%WindowsSDKLibVersion:\=%
+) ELSE (
+ SET WIN10SDKVER=%NUCRTVER%
+)
+
+REM
+REM NOTE: Check if this is the Windows Phone SDK. If so, a different batch
+REM file is necessary to setup the build environment. Since the variable
+REM values involved here may contain parenthesis, using GOTO instead of
+REM an IF block is required.
+REM
+IF DEFINED WindowsPhoneKitDir GOTO set_vcvarsall_phone
+SET VCVARSALL=%VCINSTALLDIR%\vcvarsall.bat
+GOTO set_vcvarsall_done
+:set_vcvarsall_phone
+SET VCVARSALL=%VCINSTALLDIR%\WPSDK\WP80\vcvarsphoneall.bat
+:set_vcvarsall_done
+SET VCVARSALL=%VCVARSALL:\\=\%
+
+REM
+REM NOTE: This is the outer loop. There should be exactly one iteration per
+REM platform.
+REM
+FOR %%P IN (%PLATFORMS%) DO (
+ REM
+ REM NOTE: Using the MSVC platform name, lookup the simpler platform name to
+ REM be used for the name of the platform-specific binary directory via
+ REM the environment variables setup earlier.
+ REM
+ CALL :fn_CopyVariable %%P_NAME PLATFORMNAME
+
+ REM
+ REM NOTE: This is the second loop. There should be exactly one iteration.
+ REM This loop is necessary because the PlatformName environment
+ REM variable was set above and that value is needed by some of the
+ REM commands contained in the inner loop. If these commands were
+ REM directly contained in the outer loop, the PlatformName environment
+ REM variable would be stuck with its initial empty value instead.
+ REM
+ FOR /F "tokens=2* delims==" %%D IN ('SET PLATFORMNAME') DO (
+ REM
+ REM NOTE: Attempt to clean the environment of all variables used by MSVC
+ REM and/or Visual Studio. This block may need to be updated in the
+ REM future to account for additional environment variables.
+ REM
+ CALL :fn_UnsetVariable CommandPromptType
+ CALL :fn_UnsetVariable DevEnvDir
+ CALL :fn_UnsetVariable DNX_HOME
+ CALL :fn_UnsetVariable ExtensionSdkDir
+ CALL :fn_UnsetVariable Framework35Version
+ CALL :fn_UnsetVariable Framework40Version
+ CALL :fn_UnsetVariable FrameworkDir
+ CALL :fn_UnsetVariable FrameworkDir32
+ CALL :fn_UnsetVariable FrameworkVersion
+ CALL :fn_UnsetVariable FrameworkVersion32
+ CALL :fn_UnsetVariable FSHARPINSTALLDIR
+ CALL :fn_UnsetVariable INCLUDE
+ CALL :fn_UnsetVariable LIB
+ CALL :fn_UnsetVariable LIBPATH
+ CALL :fn_UnsetVariable NETFXSDKDir
+ CALL :fn_UnsetVariable Platform
+ CALL :fn_UnsetVariable UCRTVersion
+ CALL :fn_UnsetVariable UniversalCRTSdkDir
+ REM CALL :fn_UnsetVariable VCINSTALLDIR
+ CALL :fn_UnsetVariable VSINSTALLDIR
+ CALL :fn_UnsetVariable WindowsLibPath
+ CALL :fn_UnsetVariable WindowsPhoneKitDir
+ CALL :fn_UnsetVariable WindowsSdkDir
+ CALL :fn_UnsetVariable WindowsSdkDir_35
+ CALL :fn_UnsetVariable WindowsSdkDir_old
+ CALL :fn_UnsetVariable WindowsSDKLibVersion
+ CALL :fn_UnsetVariable WindowsSDKVersion
+ CALL :fn_UnsetVariable WindowsSDK_ExecutablePath_x86
+ CALL :fn_UnsetVariable WindowsSDK_ExecutablePath_x64
+
+ REM
+ REM NOTE: Reset the PATH here to the absolute bare minimum required.
+ REM
+ CALL :fn_ResetPath
+
+ REM
+ REM NOTE: This is the inner loop. There are normally two iterations, one
+ REM for each supported build configuration, e.g. Debug or Retail.
+ REM
+ FOR %%B IN (%CONFIGURATIONS%) DO (
+ REM
+ REM NOTE: When preparing the debug build, set the DEBUG and MEMDEBUG
+ REM environment variables to be picked up by the MSVC makefile
+ REM itself.
+ REM
+ %_AECHO% Building the %%B configuration for platform %%P with name %%D...
+
+ IF /I "%%B" == "Debug" (
+ REM
+ REM NOTE: Using this level for the DEBUG environment variable should
+ REM disable all compiler optimizations and prevent use of the
+ REM NDEBUG define. Additionally, both SQLITE_ENABLE_API_ARMOR
+ REM and SQLITE_DEBUG defines should be enabled.
+ REM
+ SET DEBUG=3
+
+ REM
+ REM NOTE: Setting this to non-zero should enable the SQLITE_MEMDEBUG
+ REM define.
+ REM
+ IF NOT DEFINED NOMEMDEBUG (
+ SET MEMDEBUG=1
+ )
+ ) ELSE (
+ CALL :fn_UnsetVariable DEBUG
+ CALL :fn_UnsetVariable MEMDEBUG
+ )
+
+ REM
+ REM NOTE: Copy the extra NMAKE arguments for this configuration into the
+ REM common variable used by the actual commands.
+ REM
+ CALL :fn_CopyVariable NMAKE_ARGS_%%B NMAKE_ARGS_CFG
+
+ REM
+ REM NOTE: Launch a nested command shell to perform the following steps:
+ REM
+ REM 1. Setup the MSVC environment for this platform using the
+ REM official batch file.
+ REM
+ REM 2. Make sure that no stale build output files are present.
+ REM
+ REM 3. Build the "sqlite3.dll" and "sqlite3.lib" binaries for this
+ REM platform.
+ REM
+ REM 4. Copy the "sqlite3.dll" and "sqlite3.lib" binaries for this
+ REM platform to the platform-specific directory beneath the
+ REM binary directory.
+ REM
+ REM 5. Unless prevented from doing so, copy the "sqlite3.pdb"
+ REM symbols file for this platform to the platform-specific
+ REM directory beneath the binary directory.
+ REM
+ "%ComSpec%" /C (
+ REM
+ REM NOTE: Attempt to setup the MSVC environment for this platform.
+ REM
+ %_CECHO3% CALL "%VCVARSALL%" %%P
+ %__ECHO3% CALL "%VCVARSALL%" %%P
+
+ IF ERRORLEVEL 1 (
+ ECHO Failed to call "%VCVARSALL%" for platform %%P.
+ GOTO errors
+ )
+
+ REM
+ REM NOTE: If this batch file is not running in "what-if" mode, check to
+ REM be sure we were actually able to setup the MSVC environment
+ REM as current versions of their official batch file do not set
+ REM the exit code upon failure.
+ REM
+ IF NOT DEFINED __ECHO3 (
+ IF NOT DEFINED WindowsPhoneKitDir (
+ IF NOT DEFINED WindowsSdkDir (
+ ECHO Cannot build, Windows SDK not found for platform %%P.
+ GOTO errors
+ )
+ )
+ )
+
+ REM
+ REM NOTE: When using MSVC 2012, 2013, or 2015, the native SDK path
+ REM cannot simply be the "lib" sub-directory beneath the location
+ REM specified in the WindowsSdkDir environment variable because
+ REM that location does not actually contain the necessary library
+ REM files for x86. This must be done for each iteration because
+ REM it relies upon the WindowsSdkDir environment variable being
+ REM set by the batch file used to setup the MSVC environment.
+ REM
+ IF DEFINED SET_NSDKLIBPATH (
+ REM
+ REM NOTE: The Windows Phone SDK has a slightly different directory
+ REM structure and must be handled specially here.
+ REM
+ IF DEFINED WindowsPhoneKitDir (
+ CALL :fn_CopyVariable WindowsPhoneKitDir NSDKLIBPATH
+ CALL :fn_AppendVariable NSDKLIBPATH \lib\x86
+ ) ELSE IF DEFINED WindowsSdkDir (
+ CALL :fn_CopyVariable WindowsSdkDir NSDKLIBPATH
+
+ REM
+ REM NOTE: The Windows 8.x and Windows 10.0 SDKs have a slightly
+ REM different directory naming conventions.
+ REM
+ IF DEFINED USE_WINV100_NSDKLIBPATH (
+ CALL :fn_AppendVariable NSDKLIBPATH \..\10\lib\%WIN10SDKVER%\um\x86
+ CALL :fn_CopyVariable WindowsSdkDir PSDKLIBPATH
+ CALL :fn_AppendVariable PSDKLIBPATH lib\%WIN10SDKVER%\um\%%D
+ ) ELSE IF DEFINED USE_WINV63_NSDKLIBPATH (
+ CALL :fn_AppendVariable NSDKLIBPATH \lib\winv6.3\um\x86
+ ) ELSE IF "%VisualStudioVersion%" == "12.0" (
+ CALL :fn_AppendVariable NSDKLIBPATH \..\8.0\lib\win8\um\x86
+ ) ELSE IF "%VisualStudioVersion%" == "14.0" (
+ CALL :fn_AppendVariable NSDKLIBPATH \..\8.0\lib\win8\um\x86
+ ) ELSE (
+ CALL :fn_AppendVariable NSDKLIBPATH \lib\win8\um\x86
+ )
+ )
+ )
+
+ REM
+ REM NOTE: When using MSVC 2015, setting the Universal CRT library path
+ REM for x86 may be required as well. This must also be done for
+ REM each iteration because it relies upon the UniversalCRTSdkDir
+ REM environment variable being set by the batch file used to
+ REM setup the MSVC environment.
+ REM
+ IF DEFINED SET_NUCRTLIBPATH (
+ IF DEFINED UniversalCRTSdkDir (
+ CALL :fn_CopyVariable UniversalCRTSdkDir NUCRTLIBPATH
+ CALL :fn_AppendVariable NUCRTLIBPATH \lib\%NUCRTVER%\ucrt\x86
+ )
+ )
+
+ REM
+ REM NOTE: Unless prevented from doing so, invoke NMAKE with the MSVC
+ REM makefile to clean any stale build output from previous
+ REM iterations of this loop and/or previous runs of this batch
+ REM file, etc.
+ REM
+ IF NOT DEFINED NOCLEAN (
+ CALL :fn_MakeClean %%D
+
+ IF ERRORLEVEL 1 (
+ ECHO Failed to clean for platform %%P.
+ GOTO errors
+ )
+ ) ELSE (
+ REM
+ REM NOTE: Even when the cleaning step has been disabled, we still
+ REM need to remove the build output for all the files we are
+ REM specifically wanting to build for each platform.
+ REM
+ %_AECHO% Cleaning final core library output files only...
+ %__ECHO% DEL /Q *.lo "%DLL_FILE_NAME%" "%LIB_FILE_NAME%" "%DLL_PDB_FILE_NAME%" 2%REDIRECT% NUL
+ )
+
+ REM
+ REM NOTE: Call NMAKE with the MSVC makefile to build the "sqlite3.dll"
+ REM binary. The x86 compiler will be used to compile the native
+ REM command line tools needed during the build process itself.
+ REM Also, disable looking for and/or linking to the native Tcl
+ REM runtime library.
+ REM
+ CALL :fn_MakeDll %%D
+
+ IF ERRORLEVEL 1 (
+ ECHO Failed to build %%B "%DLL_FILE_NAME%" for platform %%P.
+ GOTO errors
+ )
+
+ REM
+ REM NOTE: Copy the "sqlite3.dll" file to the appropriate directory for
+ REM the build and platform beneath the binary directory.
+ REM
+ %__ECHO% XCOPY "%DLL_FILE_NAME%" "%BINARYDIRECTORY%\%%B\%%D\" %FFLAGS% %DFLAGS%
+
+ IF ERRORLEVEL 1 (
+ ECHO Failed to copy "%DLL_FILE_NAME%" to "%BINARYDIRECTORY%\%%B\%%D\".
+ GOTO errors
+ )
+
+ REM
+ REM NOTE: Copy the "sqlite3.lib" file to the appropriate directory for
+ REM the build and platform beneath the binary directory.
+ REM
+ %__ECHO% XCOPY "%LIB_FILE_NAME%" "%BINARYDIRECTORY%\%%B\%%D\" %FFLAGS% %DFLAGS%
+
+ IF ERRORLEVEL 1 (
+ ECHO Failed to copy "%LIB_FILE_NAME%" to "%BINARYDIRECTORY%\%%B\%%D\".
+ GOTO errors
+ )
+
+ REM
+ REM NOTE: Copy the "sqlite3.pdb" file to the appropriate directory for
+ REM the build and platform beneath the binary directory unless we
+ REM are prevented from doing so.
+ REM
+ IF NOT DEFINED NOSYMBOLS (
+ IF EXIST "%DLL_PDB_FILE_NAME%" (
+ %__ECHO% XCOPY "%DLL_PDB_FILE_NAME%" "%BINARYDIRECTORY%\%%B\%%D\" %FFLAGS% %DFLAGS%
+
+ IF ERRORLEVEL 1 (
+ ECHO Failed to copy "%DLL_PDB_FILE_NAME%" to "%BINARYDIRECTORY%\%%B\%%D\".
+ GOTO errors
+ )
+ )
+ )
+
+ REM
+ REM NOTE: If requested, also build the shell executable.
+ REM
+ IF DEFINED BUILD_ALL_SHELL (
+ REM
+ REM NOTE: If necessary, make sure any previous build output for the
+ REM shell executable is deleted.
+ REM
+ IF DEFINED NOCLEAN (
+ REM
+ REM NOTE: Even when the cleaning step has been disabled, we still
+ REM need to remove the build output for all the files we are
+ REM specifically wanting to build for each platform.
+ REM
+ %_AECHO% Cleaning final shell executable output files only...
+ %__ECHO% DEL /Q "%EXE_FILE_NAME%" "%EXE_PDB_FILE_NAME%" 2%REDIRECT% NUL
+ )
+
+ REM
+ REM NOTE: Call NMAKE with the MSVC makefile to build the "sqlite3.exe"
+ REM binary. The x86 compiler will be used to compile the native
+ REM command line tools needed during the build process itself.
+ REM Also, disable looking for and/or linking to the native Tcl
+ REM runtime library.
+ REM
+ CALL :fn_MakeExe %%D
+
+ IF ERRORLEVEL 1 (
+ ECHO Failed to build %%B "%EXE_FILE_NAME%" for platform %%P.
+ GOTO errors
+ )
+
+ REM
+ REM NOTE: Copy the "sqlite3.exe" file to the appropriate directory
+ REM for the build and platform beneath the binary directory.
+ REM
+ %__ECHO% XCOPY "%EXE_FILE_NAME%" "%BINARYDIRECTORY%\%%B\%%D\" %FFLAGS% %DFLAGS%
+
+ IF ERRORLEVEL 1 (
+ ECHO Failed to copy "%EXE_FILE_NAME%" to "%BINARYDIRECTORY%\%%B\%%D\".
+ GOTO errors
+ )
+
+ REM
+ REM NOTE: Copy the "sqlite3sh.pdb" file to the appropriate directory
+ REM for the build and platform beneath the binary directory
+ REM unless we are prevented from doing so.
+ REM
+ IF NOT DEFINED NOSYMBOLS (
+ IF EXIST "%EXE_PDB_FILE_NAME%" (
+ %__ECHO% XCOPY "%EXE_PDB_FILE_NAME%" "%BINARYDIRECTORY%\%%B\%%D\" %FFLAGS% %DFLAGS%
+
+ IF ERRORLEVEL 1 (
+ ECHO Failed to copy "%EXE_PDB_FILE_NAME%" to "%BINARYDIRECTORY%\%%B\%%D\".
+ GOTO errors
+ )
+ )
+ )
+ )
+ )
+ )
+ )
+
+ REM
+ REM NOTE: Handle any errors generated during the nested command shell.
+ REM
+ IF ERRORLEVEL 1 (
+ GOTO errors
+ )
+)
+
+REM
+REM NOTE: Restore the saved current directory from the directory stack.
+REM
+%_CECHO2% POPD
+%__ECHO2% POPD
+
+IF ERRORLEVEL 1 (
+ ECHO Could not restore directory.
+ GOTO errors
+)
+
+REM
+REM NOTE: If we get to this point, we have succeeded.
+REM
+GOTO no_errors
+
+:fn_MakeClean
+ %__ECHO% %NMAKE_CMD% clean "PLATFORM=%1" XCOMPILE=1 USE_NATIVE_LIBPATHS=1 NO_TCL=1 %NMAKE_ARGS% %NMAKE_ARGS_CFG%
+ GOTO :EOF
+
+:fn_MakeDll
+ %__ECHO% %NMAKE_CMD% "%DLL_FILE_NAME%" "PLATFORM=%1" XCOMPILE=1 USE_NATIVE_LIBPATHS=1 NO_TCL=1 %NMAKE_ARGS% %NMAKE_ARGS_CFG%
+ GOTO :EOF
+
+:fn_MakeExe
+ %__ECHO% %NMAKE_CMD% "%EXE_FILE_NAME%" "PLATFORM=%1" XCOMPILE=1 USE_NATIVE_LIBPATHS=1 NO_TCL=1 %NMAKE_ARGS% %NMAKE_ARGS_CFG%
+ GOTO :EOF
+
+:fn_ShowVariable
+ SETLOCAL
+ SET __ECHO_CMD=ECHO %%%2%%
+ FOR /F "delims=" %%V IN ('%__ECHO_CMD%') DO (
+ IF NOT "%%V" == "" (
+ IF NOT "%%V" == "%%%2%%" (
+ %_VECHO% %1 = '%%V'
+ )
+ )
+ )
+ ENDLOCAL
+ GOTO :EOF
+
+:fn_ResetErrorLevel
+ VERIFY > NUL
+ GOTO :EOF
+
+:fn_SetErrorLevel
+ VERIFY MAYBE 2> NUL
+ GOTO :EOF
+
+:fn_CopyVariable
+ IF NOT DEFINED %1 GOTO :EOF
+ IF "%2" == "" GOTO :EOF
+ SETLOCAL
+ SET __ECHO_CMD=ECHO %%%1%%
+ FOR /F "delims=" %%V IN ('%__ECHO_CMD%') DO (
+ SET VALUE=%%V
+ )
+ ENDLOCAL && SET %2=%VALUE%
+ GOTO :EOF
+
+:fn_UnsetVariable
+ SETLOCAL
+ SET VALUE=%1
+ IF DEFINED VALUE (
+ SET VALUE=
+ ENDLOCAL
+ SET %VALUE%=
+ ) ELSE (
+ ENDLOCAL
+ )
+ CALL :fn_ResetErrorLevel
+ GOTO :EOF
+
+:fn_ResetPath
+ SET PATH=%TOOLPATH%;%SystemRoot%\System32;%SystemRoot%
+ GOTO :EOF
+
+:fn_AppendVariable
+ SET __ECHO_CMD=ECHO %%%1%%
+ IF DEFINED %1 (
+ FOR /F "delims=" %%V IN ('%__ECHO_CMD%') DO (
+ SET %1=%%V%~2
+ )
+ ) ELSE (
+ SET %1=%~2
+ )
+ SET __ECHO_CMD=
+ CALL :fn_ResetErrorLevel
+ GOTO :EOF
+
+:usage
+ ECHO.
+ ECHO Usage: %~nx0 ^<binaryDirectory^>
+ ECHO.
+ GOTO errors
+
+:errors
+ CALL :fn_SetErrorLevel
+ ENDLOCAL
+ ECHO.
+ ECHO Failure, errors were encountered.
+ GOTO end_of_file
+
+:no_errors
+ CALL :fn_ResetErrorLevel
+ ENDLOCAL
+ ECHO.
+ ECHO Success, no errors were encountered.
+ GOTO end_of_file
+
+:end_of_file
+%__ECHO% EXIT /B %ERRORLEVEL%
diff --git a/tool/build-shell.sh b/tool/build-shell.sh new file mode 100644 index 0000000..a57f753 --- /dev/null +++ b/tool/build-shell.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# +# This script demonstrates how to do a full-featured build of the sqlite3 +# command-line shell on Linux. +# +# SQLite source code should be in a sibling directory named "sqlite". For +# example, put SQLite sources in ~/sqlite/sqlite and run this script from +# ~/sqlite/bld. There should be an appropriate Makefile in the current +# directory as well. +# +make sqlite3.c +gcc -o sqlite3 -g -Os -I. \ + -DSQLITE_THREADSAFE=0 \ + -DSQLITE_ENABLE_VFSTRACE \ + -DSQLITE_ENABLE_STAT3 \ + -DSQLITE_ENABLE_FTS4 \ + -DSQLITE_ENABLE_RTREE \ + -DHAVE_READLINE \ + ../sqlite/src/shell.c \ + ../sqlite/src/test_vfstrace.c \ + sqlite3.c -ldl -lreadline -lncurses diff --git a/tool/cg_anno.tcl b/tool/cg_anno.tcl new file mode 100755 index 0000000..50a3ca8 --- /dev/null +++ b/tool/cg_anno.tcl @@ -0,0 +1,79 @@ +#!/bin/sh +# \ +exec tclsh "$0" ${1+"$@"} +# +# A wrapper around cg_annotate that sets appropriate command-line options +# and rearranges the output so that annotated files occur in a consistent +# sorted order. Used by the speed-check.tcl script. +# + +set in [open "|cg_annotate --show=Ir --auto=yes --context=40 $argv" r] +set dest ! +set out(!) {} +set linenum 0 +set cntlines 0 ;# true to remember cycle counts on each line +set seenSqlite3 0 ;# true if we have seen the sqlite3.c file +while {![eof $in]} { + set line [string map {\t { }} [gets $in]] + if {[regexp {^-- Auto-annotated source: (.*)} $line all name]} { + set dest $name + if {[string match */sqlite3.c $dest]} { + set cntlines 1 + set seenSqlite3 1 + } else { + set cntlines 0 + } + } elseif {[regexp {^-- line (\d+) ------} $line all ln]} { + set line [lreplace $line 2 2 {#}] + set linenum [expr {$ln-1}] + } elseif {[regexp {^The following files chosen for } $line]} { + set dest ! + } + append out($dest) $line\n + if {$cntlines} { + incr linenum + if {[regexp {^ *([0-9,]+) } $line all x]} { + set x [string map {, {}} $x] + set cycles($linenum) $x + } + } +} +foreach x [lsort [array names out]] { + puts $out($x) +} + +# If the sqlite3.c file has been seen, then output a summary of the +# cycle counts for each file that went into making up sqlite3.c +# +if {$seenSqlite3} { + close $in + set in [open sqlite3.c] + set linenum 0 + set fn sqlite3.c + set pattern1 {^/\*+ Begin file ([^ ]+) \*} + set pattern2 {^/\*+ Continuing where we left off in ([^ ]+) \*} + while {![eof $in]} { + set line [gets $in] + incr linenum + if {[regexp $pattern1 $line all newfn]} { + set fn $newfn + } elseif {[regexp $pattern2 $line all newfn]} { + set fn $newfn + } elseif {[info exists cycles($linenum)]} { + incr fcycles($fn) $cycles($linenum) + } + } + close $in + puts {**********************************************************************} + set lx {} + set sum 0 + foreach {fn cnt} [array get fcycles] { + lappend lx [list $cnt $fn] + incr sum $cnt + } + puts [format {%20s %14d %8.3f%%} TOTAL $sum 100] + foreach entry [lsort -index 0 -integer -decreasing $lx] { + foreach {cnt fn} $entry break + puts [format {%20s %14d %8.3f%%} $fn $cnt [expr {$cnt*100.0/$sum}]] + } +} diff --git a/tool/checkSpacing.c b/tool/checkSpacing.c new file mode 100644 index 0000000..ce38b08 --- /dev/null +++ b/tool/checkSpacing.c @@ -0,0 +1,84 @@ +/* +** This program checks for formatting problems in source code: +** +** * Any use of tab characters +** * White space at the end of a line +** * Blank lines at the end of a file +** +** Any violations are reported. +*/ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#define CR_OK 0x001 +#define WSEOL_OK 0x002 + +static void checkSpacing(const char *zFile, unsigned flags){ + FILE *in = fopen(zFile, "rb"); + int i; + int seenSpace; + int seenTab; + int ln = 0; + int lastNonspace = 0; + char zLine[2000]; + if( in==0 ){ + printf("cannot open %s\n", zFile); + return; + } + while( fgets(zLine, sizeof(zLine), in) ){ + seenSpace = 0; + seenTab = 0; + ln++; + for(i=0; zLine[i]; i++){ + if( zLine[i]=='\t' && seenTab==0 ){ + printf("%s:%d: tab (\\t) character\n", zFile, ln); + seenTab = 1; + }else if( zLine[i]=='\r' ){ + if( (flags & CR_OK)==0 ){ + printf("%s:%d: carriage-return (\\r) character\n", zFile, ln); + } + }else if( zLine[i]==' ' ){ + seenSpace = 1; + }else if( zLine[i]!='\n' ){ + lastNonspace = ln; + seenSpace = 0; + } + } + if( seenSpace && (flags & WSEOL_OK)==0 ){ + printf("%s:%d: whitespace at end-of-line\n", zFile, ln); + } + } + fclose(in); + if( lastNonspace<ln ){ + printf("%s:%d: blank lines at end of file (%d)\n", + zFile, ln, ln - lastNonspace); + } +} + +int main(int argc, char **argv){ + int i; + unsigned flags = WSEOL_OK; + for(i=1; i<argc; i++){ + const char *z = argv[i]; + if( z[0]=='-' ){ + while( z[0]=='-' ) z++; + if( strcmp(z,"crok")==0 ){ + flags |= CR_OK; + }else if( strcmp(z, "wseol")==0 ){ + flags &= ~WSEOL_OK; + }else if( strcmp(z, "help")==0 ){ + printf("Usage: %s [options] FILE ...\n", argv[0]); + printf(" --crok Do not report on carriage-returns\n"); + printf(" --wseol Complain about whitespace at end-of-line\n"); + printf(" --help This message\n"); + }else{ + printf("unknown command-line option: [%s]\n", argv[i]); + printf("use --help for additional information\n"); + } + }else{ + checkSpacing(argv[i], flags); + } + } + return 0; +} diff --git a/tool/cktclsh.sh b/tool/cktclsh.sh new file mode 100644 index 0000000..1928a40 --- /dev/null +++ b/tool/cktclsh.sh @@ -0,0 +1,11 @@ +# Fail with an error if the TCLSH named in $2 is not tclsh version $1 or later. +# +echo "set vers $1" >cktclsh$1.tcl +echo 'if {$tcl_version<$vers} {exit 1}' >>cktclsh$1.tcl +if ! $2 cktclsh$1.tcl +then + echo "ERROR: This makefile target requires tclsh $1 or later." + rm cktclsh$1.tcl + exit 1 +fi +rm cktclsh$1.tcl diff --git a/tool/custom.txt b/tool/custom.txt new file mode 100644 index 0000000..b9c8a98 --- /dev/null +++ b/tool/custom.txt @@ -0,0 +1,1246 @@ +aa +aaa +abc +abcdefg +abd +abf +Abortable +acc +accessor +accum +acd +activecpu +Adaptor +Additionallly +addop +addoptrace +addr +adjustements +af +aff +afp +afterward +Agg +agg +agginfo +alikes +Alloc +alloc +alloca +allocator +allocators +alphabetics +alphanumerics +alternateform +altertab +altform +amalgamator +amongst +analyse +antipenultimate +ap +api +appdef +appendall +appendchar +appendf +ar +arg +argc +argcount +arglist +argn +args +argv +arrayname +ascii +asm +async +atoi +atomics +auth +authorizer +autocheckpoint +autocommit +autoconf +autoext +autoextension +autoinc +autoincrement +autoincremented +autoindex +autoinstall +autovac +autovacuum +autovacuumed +autovacuuming +auxdata +awk +Ax +backend +backends +backfill +backfilled +backfilling +backtrace +backtraces +backtracing +bb +bba +bcb +bcc +beginthreadex +behavior +behavioral +behaviors +benigncnt +bg +bigblob +bitcount +bitfield +bitmask +bitmasks +bitset +bitvec +bitvecs +bitwise +blobwrite +blockquote +Bloomfilter +bom +boolean +booleans +Borland +br +breadthfirstsearch +breakpoint +bt +btree +btrees +buf +bufpt +butoindex +bytearray +bytecode +bytecodevtab +byteorder +cacheflush +cachegrind +Cachesize +calc'ing +callgrind +cardinalities +cardinality +carray +cb +cd +cdbaa +ce +ceil +cellpadding +cellspacing +center +chacha +changecounter +changeset +characgter +characterset +checkpointed +Checkpointer +checkpointer +checkpointers +checkpointing +checksummed +checksums +chmod +chng +chown +chroot +chsize +chunksize +cid +cis +ckalloc'd +ckpt +cksum +cksumvfs +clientdata +closedir +clownshoe +cmd +cmp +cmpaff +Cmpr +cnt +codec +codepage +collseq +colname +compileoption +concat +config +confstr +connetion +consective +considertion +const +coredump +coroutine +coroutines +cov +crashparams +csr +csv +Cte +ctime +Ctrl +ctrl +ctx +ctype +cume +Cx +cx +Cygwin +cygwin +dan +darkstar +databasename +databse +datasource +datatypes +datetime +dbfuzz +dbinfo +dbname +dbpage +dbpagevfs +dbs +dbsize +dbsqlfuzz +dbstat +dbtotxt +De +de +deadman +deallocate +deallocated +deallocates +deallocating +deallocation +decltype +decrementing +defense +defenses +defn +defragment +defragmentation +defragmented +deinitialization +Deinitialize +deinitialize +demovfs +dependences +dequote +dequoted +dequoting +dereferenced +dereferences +desc +deserialization +deserialize +deserialized +deserializes +deserializing +dest +destructor +destructors +deterministically +dev +devsim +devsym +df +Dfdatasync +dflt +dir +directonly +dirent +dirs +disjunct +disjunction +disjuncts +diskfull +divy +dl +dll +dlopen +dlsym +docid +docids +dont +dontcache +dotfile +dotlock +doublearray +drh +dryrun +dstr +dt +Duint +dup +Durian +dword +Dx +dylib +Dyn +Ec +ec +ecel +editability +ef +efc +eg +Ei +elif +emcc +emscripten +encodings +endeavor +endfor +endian +endianness +endif +endthreadex +enum +eof +eph +Ephem +eq +eqp +equaling +equalities +errcode +errmsg +errno +errorcode +erroroffset +errstr +esc +esign +et +etfs +etilqs +eval +exe +expander +explainer +expmask +expr +exprlist +extern +fakeheap +fallocate +fanout +faultsim +favor +favors +fb +fc +fchmod +fchown +fclose +fcntl +fd +fdatasync +feb +fef +Feijoa +ferror +ffff +ffffeff +ffffffe +fffffff +ffffffff +fffffffffffffff +fflush +fg +fgets +fi +fibonacci +fid +Fifo +filecount +filectrl +filemapping +filesize +filesystem +filesystems +finalised +finalizer +findfirst +findnext +fixup +fk +fkctr +fkey +flattener +fmt +fopen +foreach +formatter +fprintf +fputs +fread +fred +fred's +freeblock +freeblocks +freelist +freepage +freespace +frombind +fs +fsanitize +fsctl +fsdir +fseek +fstat +fstree +fsync +fsynced +fsyncs +ftell +ftruncate +fts +fullfsync +fullname +fullschema +fullshm +fullsync +fullsyncs +func +funcs +fuzzer +fuzzers +fwrite +Fx +gcc +gcov +gdb +getcwd +getenv +gethostuuid +getpagesize +getpid +getrusage +getsubtype +getter +getters +gibabyte +gid +glibc +globbing +gmtime +Gosub +Goto +goto +groupbyparent +Groupid +growable +grp +hdl +hdr +hexdb +hexdouble +hexio +highwater +hijklmno +honor +honored +honoring +hostid +href +html +htsize +hw +Hwtime +icecube +ideographical +idx +idxaff +idxnum +idxstr +idx'th +ieee +ifdef +iff +ifndef +imm +impl +incr +incrblob +incrementing +indexable +indexname +infop +ing +init +initfail +initializer +initializers +initiallly +inlined +inlines +inlining +ino +inode +inodes +inopertune +installable +intarray +inteface +interoperate +interp +interpretable +intkey +intptr +intreal +intrinsics +invalidations +invariants +io +ioerr +iotrace +ipk +iplan +isalnum +isalpha +isatty +isdigit +isempty +isexplain +ismemdb +isnan +isspace +isxdigit +i'th +iversion +jfd +jj +jointype +jointypes +journaled +journaling +journalled +journalling +journalmode +jrnl +jsl +json +jt +julian +julianday +keyinfo +keywordhash +kibibytes +kvstorage +kvvfs +lappend +lasterrno +lbl +ldl +le +leafdata +leftjustify +len +leveling +lexeme +lexemes +lhs +li +libm +libversion +lifecycle +lindex +lineno +Linenoise +linux +lised +lld +llvm +lm +ln +lncurses +loadext +localhost +localtime +lockd +lockdata +lockingcontext +lockless +lockproxy +locktype +logmsg +longvalue +longwords +lookahead +Lookaside +lookaside +lookups +losslessly +lpthread +lrc +lreadline +lru +lseek +lt +lvalue +lwr +makefile +makefiles +malloc +malloc'd +malloced +mallocs +manditory +manpage +matchinfo +materializations +mathfuncs +maxsize +mbcs +Mcafee +md +Meeus +mem +memcmp +memcpy +memdb +memdebug +memget +memmove +mempool +memset +memstatus +memsys +memvfs +mergeable +mergepatch +middleware +millisec +mincore +mingw +mis +miscoded +mj +mkctimec +mkdir +mkkeywordhash +mkopcodec +mkopcodeh +mkpragmatab +mmap +mmapped +mno +modeof +Movepage +mprintf +msd +msdos +msec +msg +msgs +msize +msvc +mtime +mult +multibyte +multiplex'ed +multiplexor +multithreaded +multiwrite +mutex +mutexes +mutexing +mx +mxpathname +myprefix +nal +namecontext +namespace +natively +nbr +nbsp +ncell +ncol +ncycle +nd +ndlt +neighbors +neq +nestable +newrowid +nfs +nlt +nnn +nocase +nochange +nochng +nofollow +nolock +nomem +nomutex +nonblocking +nonroot +noop +noshm +notational +notheld +notnull +nowrap +nr +ntile +nul +nullable +nullif +nullvalue +Num +objproc +oc +offsetof +ofst +ogham +oid +Ok +ok +ol +onecolumn +onepass +onoff +onwards +oom +opcodesnever +openable +opendir +optimizers +optimizible +orconf +orderby +orderbylist +os +Oswrite +overread +overreads +overrideable +oversize +overwriteable +ovewrite +ovfl +pagecache +pagecaches +pagecount +pagelist +pageno +pagesize +pagetype +Param +params +passwd +patchset +pathname +pathnames +pc +pcache +pclose +pcx +pgno +pgoffset +pgsize +pgsz +pid +pluggable +pmasz +pn +popen +pos +posix +Postgres +Powersafe +powersafe +pq +pqr +pqrstuvw +pragma +pragmaed +pragma's +pragmas +pre +pread +preallocate +preallocated +precisions +precompiled +precomputed +prefetch +prefetched +preformated +preformatted +prepend +prepended +prepending +prepends +prepopulate +preprocess +preprocessed +preprocessing +preprocessor +prereq +prereqs +preupdate +primarykey +printf +printfs +prng +proc +procs +profiler +proleptic +proxying +proxys +pseudocode +pseudotable +psow +psz +pthread +pthreads +ptr +ptrmap +ptrs +purgeable +putsnl +pwrite +pz +qbox +Qcircle +quotaed +quotefix +radix +randomblob +randstr +rarr +rc +rcauth +Rcvr +rds +readdir +readline +readlock +readonly +Readr +realloc +reallocs +realvalue +rebalance +rebalanced +recoverability +redefinable +redux +reenabled +reentrant +refcount +refcounts +regs +reindexed +reinitializer +reinitializes +rekey +rekeyed +relevancies +relink +relink +relock +renormalize +reoptimize +reparse +reparse +reparsed +reparsing +reportable +reprepare +reprepare +reprepared +reprepares +representable +repurpose +Req'd +requeries +requote +reregister +reseek +reservebytes +resumable +retarget +retargeted +retrys +returntype +rfc +rhs +Ri +Rivest +ro +rootpage +rowid +rowids +Rowkey +rownumber +rowset +runtime +rw +rwc +samplelib +sampleno +sandboxed +sandboxing +savepoint +savepoints +scanf +scanstats +scanstatus +schemas +sectorsize +selecttrace +selftest +setrlimit +setsubtype +settitle +sharable +shm +shmlock +sibs +sig +signaling +significand +sizehint +Sizeof +sizeof +snprintf +Solaris +sorterref +soundex +sourceid +speedtest +sprintf +sql +sqlar +sqlite +sqliteplocks +sqliterc +sqllog +sqllogglobal +sqlthread +sqr +sqrt +src +srcck +statfs +stderr +stdin +stdout +stmt +stmts +str +strace +strcasecmp +strcmp +strdup +strerror +strftime +strglob +stricmp +stringify +strlen +strlike +strncmp +strncpy +strnicmp +stronly +strstr +struct +structs +subbitmap +subcases +subclassed +subclauses +subcomponents +subdirectory +subelement +subexpression +subexpressions +subfunction +subfunctions +subitem +subjournals +sublevels +subnode +suboptimal +subpages +subprocedures +subprog +subprogs +subq +subqueries +subquery +Subrtn +subsec +subsecond +subsequence +substr +substring +substrings +subsubstructure +subsubterms +Subtask +subtasks +subterm +subterms +subtransaction +subtransactions +subtree +subtrees +subtype +subtypes +sumint +superclass +superlock +superset +superunlock +symlinks +synching +sys +syscall +sz +szosfile +tablename +tailval +tailvar +tbl +tblname +Tcl +tcl +Tclcmd +tclcmd +tclsh +tclsqlite +tclvar +td +tempfilename +testcase +testctrl +testfixture +testtag +testvfs +textarray +th +threadid +threadsafe +threadsafety +throughs +tht +timediff +tkt +tm +tmp +tmpdir +tmpfs +tnum +Todo +tokenize +tokenizer +tokenizing +tolower +toobig +toupper +treetrace +treeview +trimleft +trimright +truesize +trys +Tsd +Ts'o +tunable +tvfs +txn +txt +Typecheck +typedef +typedefed +typedefs +typename +typenames +typeof +tz +uber +uid +uint +ul +umask +Un +un +unallocated +unanalyzed +unary +unbuffered +unclosed +uncompiled +uncomputed +undefining +underfull +unexpanded +unfinalized +unfreed +unhex +unicode +unindexed +uninit +uninitialize +unintuitive +unioned +unissued +unix +unixepoch +unlink +unlinked +unlinking +unlinks +unmap +unmapped +unmapping +unoptimized +unparsed +unreduced +unref +unreferenced +unrefs +unregister +unregistering +unregisters +unresolvable +unsynced +unterminated +untracked +untrusted +Upfrom +uppercasing +upr +Upsert +upsert +upto +uptr +uri +userauth +userdata +Userid +usleep +utc +Utf +utf +util +uuu +uuuuu +uuzzzz +va +valgrind +vanishingly +vappendf +vararg +varargs +varint +varints +varname +vcolumn +vdbe +vdbeapi +vdbe's +vdbes +vdbesort +ve +verifications +vfs +vfslog +vfsname +vfs's +vfstrace +vm +vmprintf +vmstep +vsnprintf +vt +vtab +vtabs +Vugt +vvv +vvvv +vvvvv +vvvvvv +vwait +vxworks +wal +wasm +wherecode +whereexpr +wheretrace +whitespace +Willmann +withoutrowid +wr +wrapup +writeable +writecrash +writefile +wsd +ww +wwww +wwzzzzyy +wxyz +xa +xac +xb +xbf +xc +xd +xdg +xe +xf +xfe +xfer +xff +xfff +xfffd +xfffe +xffffffff +x'hhhhhh +xinfo +xlc +xtype +xxxx +xxxxx +xxxxxx +xxxxxxx +xxxxxxxx +xyz +xyzzy +yy +yyxxxxxx +yyy +yyyyy +yyyyyy +zeroblob +Zeroblobs +zerodata +zeropad +zipfile +zipvfs +zplan +zulu +zzzz +zzzzyyyy diff --git a/tool/dbhash.c b/tool/dbhash.c new file mode 100644 index 0000000..78685dc --- /dev/null +++ b/tool/dbhash.c @@ -0,0 +1,491 @@ +/* +** 2016-06-07 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This is a utility program that computes an SHA1 hash on the content +** of an SQLite database. +** +** The hash is computed over just the content of the database. Free +** space inside of the database file, and alternative on-disk representations +** of the same content (ex: UTF8 vs UTF16) do not affect the hash. So, +** for example, the database file page size, encoding, and auto_vacuum setting +** can all be changed without changing the hash. +*/ +#include <stdio.h> +#include <stdlib.h> +#include <stdarg.h> +#include <ctype.h> +#include <string.h> +#include <assert.h> +#include "sqlite3.h" + +/* Context for the SHA1 hash */ +typedef struct SHA1Context SHA1Context; +struct SHA1Context { + unsigned int state[5]; + unsigned int count[2]; + unsigned char buffer[64]; +}; + +/* +** All global variables are gathered into the "g" singleton. +*/ +struct GlobalVars { + const char *zArgv0; /* Name of program */ + unsigned fDebug; /* Debug flags */ + sqlite3 *db; /* The database connection */ + SHA1Context cx; /* SHA1 hash context */ +} g; + +/* +** Debugging flags +*/ +#define DEBUG_FULLTRACE 0x00000001 /* Trace hash to stderr */ + +/****************************************************************************** +** The Hash Engine +** +** Modify these routines (and appropriate state fields in global variable 'g') +** in order to compute a different (better?) hash of the database. +*/ +/* + * blk0() and blk() perform the initial expand. + * I got the idea of expanding during the round function from SSLeay + * + * blk0le() for little-endian and blk0be() for big-endian. + */ +#define SHA_ROT(x,l,r) ((x) << (l) | (x) >> (r)) +#define rol(x,k) SHA_ROT(x,k,32-(k)) +#define ror(x,k) SHA_ROT(x,32-(k),k) + +#define blk0le(i) (block[i] = (ror(block[i],8)&0xFF00FF00) \ + |(rol(block[i],8)&0x00FF00FF)) +#define blk0be(i) block[i] +#define blk(i) (block[i&15] = rol(block[(i+13)&15]^block[(i+8)&15] \ + ^block[(i+2)&15]^block[i&15],1)) + +/* + * (R0+R1), R2, R3, R4 are the different operations (rounds) used in SHA1 + * + * Rl0() for little-endian and Rb0() for big-endian. Endianness is + * determined at run-time. + */ +#define Rl0(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk0le(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define Rb0(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk0be(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define R1(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define R2(v,w,x,y,z,i) \ + z+=(w^x^y)+blk(i)+0x6ED9EBA1+rol(v,5);w=ror(w,2); +#define R3(v,w,x,y,z,i) \ + z+=(((w|x)&y)|(w&x))+blk(i)+0x8F1BBCDC+rol(v,5);w=ror(w,2); +#define R4(v,w,x,y,z,i) \ + z+=(w^x^y)+blk(i)+0xCA62C1D6+rol(v,5);w=ror(w,2); + +/* + * Hash a single 512-bit block. This is the core of the algorithm. + */ +#define a qq[0] +#define b qq[1] +#define c qq[2] +#define d qq[3] +#define e qq[4] + +void SHA1Transform(unsigned int state[5], const unsigned char buffer[64]){ + unsigned int qq[5]; /* a, b, c, d, e; */ + static int one = 1; + unsigned int block[16]; + memcpy(block, buffer, 64); + memcpy(qq,state,5*sizeof(unsigned int)); + + /* Copy g.cx.state[] to working vars */ + /* + a = state[0]; + b = state[1]; + c = state[2]; + d = state[3]; + e = state[4]; + */ + + /* 4 rounds of 20 operations each. Loop unrolled. */ + if( 1 == *(unsigned char*)&one ){ + Rl0(a,b,c,d,e, 0); Rl0(e,a,b,c,d, 1); Rl0(d,e,a,b,c, 2); Rl0(c,d,e,a,b, 3); + Rl0(b,c,d,e,a, 4); Rl0(a,b,c,d,e, 5); Rl0(e,a,b,c,d, 6); Rl0(d,e,a,b,c, 7); + Rl0(c,d,e,a,b, 8); Rl0(b,c,d,e,a, 9); Rl0(a,b,c,d,e,10); Rl0(e,a,b,c,d,11); + Rl0(d,e,a,b,c,12); Rl0(c,d,e,a,b,13); Rl0(b,c,d,e,a,14); Rl0(a,b,c,d,e,15); + }else{ + Rb0(a,b,c,d,e, 0); Rb0(e,a,b,c,d, 1); Rb0(d,e,a,b,c, 2); Rb0(c,d,e,a,b, 3); + Rb0(b,c,d,e,a, 4); Rb0(a,b,c,d,e, 5); Rb0(e,a,b,c,d, 6); Rb0(d,e,a,b,c, 7); + Rb0(c,d,e,a,b, 8); Rb0(b,c,d,e,a, 9); Rb0(a,b,c,d,e,10); Rb0(e,a,b,c,d,11); + Rb0(d,e,a,b,c,12); Rb0(c,d,e,a,b,13); Rb0(b,c,d,e,a,14); Rb0(a,b,c,d,e,15); + } + R1(e,a,b,c,d,16); R1(d,e,a,b,c,17); R1(c,d,e,a,b,18); R1(b,c,d,e,a,19); + R2(a,b,c,d,e,20); R2(e,a,b,c,d,21); R2(d,e,a,b,c,22); R2(c,d,e,a,b,23); + R2(b,c,d,e,a,24); R2(a,b,c,d,e,25); R2(e,a,b,c,d,26); R2(d,e,a,b,c,27); + R2(c,d,e,a,b,28); R2(b,c,d,e,a,29); R2(a,b,c,d,e,30); R2(e,a,b,c,d,31); + R2(d,e,a,b,c,32); R2(c,d,e,a,b,33); R2(b,c,d,e,a,34); R2(a,b,c,d,e,35); + R2(e,a,b,c,d,36); R2(d,e,a,b,c,37); R2(c,d,e,a,b,38); R2(b,c,d,e,a,39); + R3(a,b,c,d,e,40); R3(e,a,b,c,d,41); R3(d,e,a,b,c,42); R3(c,d,e,a,b,43); + R3(b,c,d,e,a,44); R3(a,b,c,d,e,45); R3(e,a,b,c,d,46); R3(d,e,a,b,c,47); + R3(c,d,e,a,b,48); R3(b,c,d,e,a,49); R3(a,b,c,d,e,50); R3(e,a,b,c,d,51); + R3(d,e,a,b,c,52); R3(c,d,e,a,b,53); R3(b,c,d,e,a,54); R3(a,b,c,d,e,55); + R3(e,a,b,c,d,56); R3(d,e,a,b,c,57); R3(c,d,e,a,b,58); R3(b,c,d,e,a,59); + R4(a,b,c,d,e,60); R4(e,a,b,c,d,61); R4(d,e,a,b,c,62); R4(c,d,e,a,b,63); + R4(b,c,d,e,a,64); R4(a,b,c,d,e,65); R4(e,a,b,c,d,66); R4(d,e,a,b,c,67); + R4(c,d,e,a,b,68); R4(b,c,d,e,a,69); R4(a,b,c,d,e,70); R4(e,a,b,c,d,71); + R4(d,e,a,b,c,72); R4(c,d,e,a,b,73); R4(b,c,d,e,a,74); R4(a,b,c,d,e,75); + R4(e,a,b,c,d,76); R4(d,e,a,b,c,77); R4(c,d,e,a,b,78); R4(b,c,d,e,a,79); + + /* Add the working vars back into context.state[] */ + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; + state[4] += e; +} + + +/* Initialize the SHA1 hash */ +static void hash_init(void){ + /* SHA1 initialization constants */ + g.cx.state[0] = 0x67452301; + g.cx.state[1] = 0xEFCDAB89; + g.cx.state[2] = 0x98BADCFE; + g.cx.state[3] = 0x10325476; + g.cx.state[4] = 0xC3D2E1F0; + g.cx.count[0] = g.cx.count[1] = 0; +} + +/* Add new content to the SHA1 hash */ +static void hash_step(const unsigned char *data, unsigned int len){ + unsigned int i, j; + + j = g.cx.count[0]; + if( (g.cx.count[0] += len << 3) < j ){ + g.cx.count[1] += (len>>29)+1; + } + j = (j >> 3) & 63; + if( (j + len) > 63 ){ + (void)memcpy(&g.cx.buffer[j], data, (i = 64-j)); + SHA1Transform(g.cx.state, g.cx.buffer); + for(; i + 63 < len; i += 64){ + SHA1Transform(g.cx.state, &data[i]); + } + j = 0; + }else{ + i = 0; + } + (void)memcpy(&g.cx.buffer[j], &data[i], len - i); +} + + +/* Add padding and compute and output the message digest. */ +static void hash_finish(const char *zName){ + unsigned int i; + unsigned char finalcount[8]; + unsigned char digest[20]; + static const char zEncode[] = "0123456789abcdef"; + char zOut[41]; + + for (i = 0; i < 8; i++){ + finalcount[i] = (unsigned char)((g.cx.count[(i >= 4 ? 0 : 1)] + >> ((3-(i & 3)) * 8) ) & 255); /* Endian independent */ + } + hash_step((const unsigned char *)"\200", 1); + while ((g.cx.count[0] & 504) != 448){ + hash_step((const unsigned char *)"\0", 1); + } + hash_step(finalcount, 8); /* Should cause a SHA1Transform() */ + for (i = 0; i < 20; i++){ + digest[i] = (unsigned char)((g.cx.state[i>>2] >> ((3-(i & 3)) * 8) ) & 255); + } + for(i=0; i<20; i++){ + zOut[i*2] = zEncode[(digest[i]>>4)&0xf]; + zOut[i*2+1] = zEncode[digest[i] & 0xf]; + } + zOut[i*2]= 0; + printf("%s %s\n", zOut, zName); +} +/* End of the hashing logic +*******************************************************************************/ + +/* +** Print an error resulting from faulting command-line arguments and +** abort the program. +*/ +static void cmdlineError(const char *zFormat, ...){ + va_list ap; + fprintf(stderr, "%s: ", g.zArgv0); + va_start(ap, zFormat); + vfprintf(stderr, zFormat, ap); + va_end(ap); + fprintf(stderr, "\n\"%s --help\" for more help\n", g.zArgv0); + exit(1); +} + +/* +** Print an error message for an error that occurs at runtime, then +** abort the program. +*/ +static void runtimeError(const char *zFormat, ...){ + va_list ap; + fprintf(stderr, "%s: ", g.zArgv0); + va_start(ap, zFormat); + vfprintf(stderr, zFormat, ap); + va_end(ap); + fprintf(stderr, "\n"); + exit(1); +} + +/* +** Prepare a new SQL statement. Print an error and abort if anything +** goes wrong. +*/ +static sqlite3_stmt *db_vprepare(const char *zFormat, va_list ap){ + char *zSql; + int rc; + sqlite3_stmt *pStmt; + + zSql = sqlite3_vmprintf(zFormat, ap); + if( zSql==0 ) runtimeError("out of memory"); + rc = sqlite3_prepare_v2(g.db, zSql, -1, &pStmt, 0); + if( rc ){ + runtimeError("SQL statement error: %s\n\"%s\"", sqlite3_errmsg(g.db), + zSql); + } + sqlite3_free(zSql); + return pStmt; +} +static sqlite3_stmt *db_prepare(const char *zFormat, ...){ + va_list ap; + sqlite3_stmt *pStmt; + va_start(ap, zFormat); + pStmt = db_vprepare(zFormat, ap); + va_end(ap); + return pStmt; +} + +/* +** Compute the hash for all rows of the query formed from the printf-style +** zFormat and its argument. +*/ +static void hash_one_query(const char *zFormat, ...){ + va_list ap; + sqlite3_stmt *pStmt; /* The query defined by zFormat and "..." */ + int nCol; /* Number of columns in the result set */ + int i; /* Loop counter */ + + /* Prepare the query defined by zFormat and "..." */ + va_start(ap, zFormat); + pStmt = db_vprepare(zFormat, ap); + va_end(ap); + nCol = sqlite3_column_count(pStmt); + + /* Compute a hash over the result of the query */ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + for(i=0; i<nCol; i++){ + switch( sqlite3_column_type(pStmt,i) ){ + case SQLITE_NULL: { + hash_step((const unsigned char*)"0",1); + if( g.fDebug & DEBUG_FULLTRACE ) fprintf(stderr, "NULL\n"); + break; + } + case SQLITE_INTEGER: { + sqlite3_uint64 u; + int j; + unsigned char x[8]; + sqlite3_int64 v = sqlite3_column_int64(pStmt,i); + memcpy(&u, &v, 8); + for(j=7; j>=0; j--){ + x[j] = u & 0xff; + u >>= 8; + } + hash_step((const unsigned char*)"1",1); + hash_step(x,8); + if( g.fDebug & DEBUG_FULLTRACE ){ + fprintf(stderr, "INT %s\n", sqlite3_column_text(pStmt,i)); + } + break; + } + case SQLITE_FLOAT: { + sqlite3_uint64 u; + int j; + unsigned char x[8]; + double r = sqlite3_column_double(pStmt,i); + memcpy(&u, &r, 8); + for(j=7; j>=0; j--){ + x[j] = u & 0xff; + u >>= 8; + } + hash_step((const unsigned char*)"2",1); + hash_step(x,8); + if( g.fDebug & DEBUG_FULLTRACE ){ + fprintf(stderr, "FLOAT %s\n", sqlite3_column_text(pStmt,i)); + } + break; + } + case SQLITE_TEXT: { + int n = sqlite3_column_bytes(pStmt, i); + const unsigned char *z = sqlite3_column_text(pStmt, i); + hash_step((const unsigned char*)"3", 1); + hash_step(z, n); + if( g.fDebug & DEBUG_FULLTRACE ){ + fprintf(stderr, "TEXT '%s'\n", sqlite3_column_text(pStmt,i)); + } + break; + } + case SQLITE_BLOB: { + int n = sqlite3_column_bytes(pStmt, i); + const unsigned char *z = sqlite3_column_blob(pStmt, i); + hash_step((const unsigned char*)"4", 1); + hash_step(z, n); + if( g.fDebug & DEBUG_FULLTRACE ){ + fprintf(stderr, "BLOB (%d bytes)\n", n); + } + break; + } + } + } + } + sqlite3_finalize(pStmt); +} + + +/* +** Print sketchy documentation for this utility program +*/ +static void showHelp(void){ + printf("Usage: %s [options] FILE ...\n", g.zArgv0); + printf( +"Compute a SHA1 hash on the content of database FILE. System tables such as\n" +"sqlite_stat1, sqlite_stat4, and sqlite_sequence are omitted from the hash.\n" +"Options:\n" +" --debug N Set debugging flags to N (experts only)\n" +" --like PATTERN Only hash tables whose name is LIKE the pattern\n" +" --schema-only Only hash the schema - omit table content\n" +" --without-schema Only hash table content - omit the schema\n" + ); +} + +int main(int argc, char **argv){ + const char *zDb = 0; /* Name of the database currently being hashed */ + int i; /* Loop counter */ + int rc; /* Subroutine return code */ + char *zErrMsg; /* Error message when opening database */ + sqlite3_stmt *pStmt; /* An SQLite query */ + const char *zLike = 0; /* LIKE pattern of tables to hash */ + int omitSchema = 0; /* True to compute hash on content only */ + int omitContent = 0; /* True to compute hash on schema only */ + int nFile = 0; /* Number of input filenames seen */ + + g.zArgv0 = argv[0]; + sqlite3_config(SQLITE_CONFIG_SINGLETHREAD); + for(i=1; i<argc; i++){ + const char *z = argv[i]; + if( z[0]=='-' ){ + z++; + if( z[0]=='-' ) z++; + if( strcmp(z,"debug")==0 ){ + if( i==argc-1 ) cmdlineError("missing argument to %s", argv[i]); + g.fDebug = strtol(argv[++i], 0, 0); + }else + if( strcmp(z,"help")==0 ){ + showHelp(); + return 0; + }else + if( strcmp(z,"like")==0 ){ + if( i==argc-1 ) cmdlineError("missing argument to %s", argv[i]); + if( zLike!=0 ) cmdlineError("only one --like allowed"); + zLike = argv[++i]; + }else + if( strcmp(z,"schema-only")==0 ){ + omitContent = 1; + }else + if( strcmp(z,"without-schema")==0 ){ + omitSchema = 1; + }else + { + cmdlineError("unknown option: %s", argv[i]); + } + }else{ + nFile++; + if( nFile<i ) argv[nFile] = argv[i]; + } + } + if( nFile==0 ){ + cmdlineError("no input files specified - nothing to do"); + } + if( omitSchema && omitContent ){ + cmdlineError("only one of --without-schema and --omit-schema allowed"); + } + if( zLike==0 ) zLike = "%"; + + for(i=1; i<=nFile; i++){ + static const int openFlags = + SQLITE_OPEN_READWRITE | /* Read/write so hot journals can recover */ + SQLITE_OPEN_URI + ; + zDb = argv[i]; + rc = sqlite3_open_v2(zDb, &g.db, openFlags, 0); + if( rc ){ + fprintf(stderr, "cannot open database file '%s'\n", zDb); + continue; + } + rc = sqlite3_exec(g.db, "SELECT * FROM sqlite_schema", 0, 0, &zErrMsg); + if( rc || zErrMsg ){ + sqlite3_close(g.db); + g.db = 0; + fprintf(stderr, "'%s' is not a valid SQLite database\n", zDb); + continue; + } + + /* Start the hash */ + hash_init(); + + /* Hash table content */ + if( !omitContent ){ + pStmt = db_prepare( + "SELECT name FROM sqlite_schema\n" + " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n" + " AND name NOT LIKE 'sqlite_%%'\n" + " AND name LIKE '%q'\n" + " ORDER BY name COLLATE nocase;\n", + zLike + ); + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + /* We want rows of the table to be hashed in PRIMARY KEY order. + ** Technically, an ORDER BY clause is required to guarantee that + ** order. However, though not guaranteed by the documentation, every + ** historical version of SQLite has always output rows in PRIMARY KEY + ** order when there is no WHERE or GROUP BY clause, so the ORDER BY + ** can be safely omitted. */ + hash_one_query("SELECT * FROM \"%w\"", sqlite3_column_text(pStmt,0)); + } + sqlite3_finalize(pStmt); + } + + /* Hash the database schema */ + if( !omitSchema ){ + hash_one_query( + "SELECT type, name, tbl_name, sql FROM sqlite_schema\n" + " WHERE tbl_name LIKE '%q'\n" + " ORDER BY name COLLATE nocase;\n", + zLike + ); + } + + /* Finish and output the hash and close the database connection. */ + hash_finish(zDb); + sqlite3_close(g.db); + } + return 0; +} diff --git a/tool/dbtotxt.c b/tool/dbtotxt.c new file mode 100644 index 0000000..fbd6e3d --- /dev/null +++ b/tool/dbtotxt.c @@ -0,0 +1,188 @@ +/* +** Copyright 2008 D. Richard Hipp and Hipp, Wyrick & Company, Inc. +** All Rights Reserved +** +****************************************************************************** +** +** This file implements a stand-alone utility program that converts +** a binary file (usually an SQLite database) into a text format that +** is compact and friendly to human-readers. +** +** Usage: +** +** dbtotxt [OPTIONS] FILENAME +** +** where OPTIONS are zero or more of: +** +** --for-cli prepending '.open --hexdb' to the output +** +** --script The input file is expected to start with a +** zero-terminated SQL string. Output the +** ".open --hexdb" header, then the database +** then the SQL. +** +** --pagesize N set the database page size for later reading +** +** The translation of the database appears on standard output. If the +** --pagesize command-line option is omitted, then the page size is taken +** from the database header. +** +** Compactness is achieved by suppressing lines of all zero bytes. This +** works well at compressing test databases that are mostly empty. But +** the output will probably be lengthy for a real database containing lots +** of real content. For maximum compactness, it is suggested that test +** databases be constructed with "zeroblob()" rather than "randomblob()" +** used for filler content and with "PRAGMA secure_delete=ON" selected to +** zero-out deleted content. +*/ +#include <stdio.h> +#include <string.h> +#include <stdlib.h> +#include <ctype.h> + +/* Return true if the line is all zeros */ +static int allZero(unsigned char *aLine){ + int i; + for(i=0; i<16 && aLine[i]==0; i++){} + return i==16; +} + +int main(int argc, char **argv){ + int pgsz = 0; /* page size */ + int forCli = 0; /* whether to prepend with .open */ + int bSQL = 0; /* Expect and SQL prefix */ + long szFile; /* Size of the input file in bytes */ + FILE *in; /* Input file */ + int nSQL; /* Number of bytes of script */ + int i, j; /* Loop counters */ + int nErr = 0; /* Number of errors */ + const char *zInputFile = 0; /* Name of the input file */ + const char *zBaseName = 0; /* Base name of the file */ + int lastPage = 0; /* Last page number shown */ + int iPage; /* Current page number */ + unsigned char *aData = 0; /* All data */ + unsigned char *aLine; /* A single line of the file */ + unsigned char *aHdr; /* File header */ + unsigned char bShow[256]; /* Characters ok to display */ + memset(bShow, '.', sizeof(bShow)); + for(i=' '; i<='~'; i++){ + if( i!='{' && i!='}' && i!='"' && i!='\\' ) bShow[i] = (unsigned char)i; + } + for(i=1; i<argc; i++){ + if( argv[i][0]=='-' ){ + const char *z = argv[i]; + z++; + if( z[0]=='-' ) z++; + if( strcmp(z,"pagesize")==0 ){ + i++; + pgsz = atoi(argv[i]); + if( pgsz<512 || pgsz>65536 || (pgsz&(pgsz-1))!=0 ){ + fprintf(stderr, "Page size must be a power of two between" + " 512 and 65536.\n"); + nErr++; + } + continue; + }else if( strcmp(z,"for-cli")==0 ){ + forCli = 1; + continue; + }else if( strcmp(z,"script")==0 ){ + forCli = 1; + bSQL = 1; + continue; + } + fprintf(stderr, "Unknown option: %s\n", argv[i]); + nErr++; + }else if( zInputFile ){ + fprintf(stderr, "Already using a different input file: [%s]\n", argv[i]); + nErr++; + }else{ + zInputFile = argv[i]; + } + } + if( zInputFile==0 ){ + fprintf(stderr, "No input file specified.\n"); + nErr++; + } + if( nErr ){ + fprintf(stderr, + "Usage: %s [--pagesize N] [--script] [--for-cli] FILENAME\n", argv[0]); + exit(1); + } + in = fopen(zInputFile, "rb"); + if( in==0 ){ + fprintf(stderr, "Cannot open input file [%s]\n", zInputFile); + exit(1); + } + fseek(in, 0, SEEK_END); + szFile = ftell(in); + rewind(in); + if( szFile<100 ){ + fprintf(stderr, "File too short. Minimum size is 100 bytes.\n"); + exit(1); + } + aData = malloc( szFile+16 ); + if( aData==0 ){ + fprintf(stderr, "Failed to allocate %ld bytes\n", szFile); + exit(1); + } + if( fread(aData, szFile, 1, in)!=1 ){ + fprintf(stderr, "Cannot read file info memory\n"); + exit(1); + } + memset(aData+szFile, 0, 16); + fclose(in); + if( bSQL ){ + for(i=0; i<szFile && aData[i]!=0; i++){} + if( i==szFile ){ + fprintf(stderr, "No zero terminator on SQL script\n"); + exit(1); + } + nSQL = i+1; + if( szFile - nSQL<100 ){ + fprintf(stderr, "Less than 100 bytes in the database\n"); + exit(1); + } + }else{ + nSQL = 0; + } + aHdr = aData + nSQL; + if( pgsz==0 ){ + pgsz = (aHdr[16]<<8) | aHdr[17]; + if( pgsz==1 ) pgsz = 65536; + if( pgsz<512 || (pgsz&(pgsz-1))!=0 ){ + fprintf(stderr, "Invalid page size in header: %d\n", pgsz); + exit(1); + } + } + zBaseName = zInputFile; + for(i=0; zInputFile[i]; i++){ + if( zInputFile[i]=='/' && zInputFile[i+1]!=0 ) zBaseName = zInputFile+i+1; + } + if( forCli ){ + printf(".open --hexdb\n"); + } + printf("| size %d pagesize %d filename %s\n",(int)szFile,pgsz,zBaseName); + for(i=nSQL; i<szFile; i+=16){ + aLine = aData+i; + if( allZero(aLine) ) continue; + iPage = i/pgsz + 1; + if( lastPage!=iPage ){ + printf("| page %d offset %d\n", iPage, (iPage-1)*pgsz); + lastPage = iPage; + } + printf("| %5d:", i-(iPage-1)*pgsz); + for(j=0; j<16; j++) printf(" %02x", aLine[j]); + printf(" "); + for(j=0; j<16; j++){ + unsigned char c = (unsigned char)aLine[j]; + fputc( bShow[c], stdout); + } + fputc('\n', stdout); + } + printf("| end %s\n", zBaseName); + if( nSQL>0 ){ + printf("%s\n", aData); + } + free( aData ); + return 0; +} diff --git a/tool/dbtotxt.md b/tool/dbtotxt.md new file mode 100644 index 0000000..f2bd7c9 --- /dev/null +++ b/tool/dbtotxt.md @@ -0,0 +1,56 @@ +<h1 align="center">The dbtotxt Tool</h1> + +The dbtotxt utility program reads an SQLite database file and writes its +raw binary content to screen as a hex dump for testing and debugging +purposes. + +The hex-dump output is formatted in such a way as to be easily readable +both by humans and by software. The dbtotxt utility has long been a part +of the TH3 test suite. The output of dbtotxt can be embedded in TH3 test +scripts and used to generate very specific database files, perhaps with +deliberately introduced corruption. The cov1/corrupt*.test modules in +TH3 make extensive use of dbtotxt. + +More recently (2018-12-13) the dbtotxt utility has been added to the SQLite +core and the command-line shell (CLI) has been augmented to be able to read +dbtotxt output. The CLI dot-command is: + +> .open --hexdb ?OPTIONAL-FILENAME? + +If the OPTIONAL-FILENAME is included, then content is read from that file. +If OPTIONAL-FILENAME is omitted, then the text is taken from the input stream, +terminated by the "| end" line of the dbtotxt text. This allows small test +databases to be embedded directly in scripts. Consider this example: + +> + .open --hexdb + | size 8192 pagesize 4096 filename x9.db + | page 1 offset 0 + | 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. + | 16: 10 00 01 01 00 40 20 20 00 00 00 04 00 00 00 02 .....@ ........ + | 32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04 ................ + | 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ + | 80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 ................ + | 96: 00 2e 30 38 0d 00 00 00 01 0f c0 00 0f c0 00 00 ..08............ + | 4032: 3e 01 06 17 11 11 01 69 74 61 62 6c 65 74 31 74 >......itablet1t + | 4048: 31 02 43 52 45 41 54 45 20 54 41 42 4c 45 20 74 1.CREATE TABLE t + | 4064: 31 28 78 2c 79 20 44 45 46 41 55 4c 54 20 78 27 1(x,y DEFAULT x' + | 4080: 66 66 27 2c 7a 20 44 45 46 41 55 4c 54 20 30 29 ff',z DEFAULT 0) + | page 2 offset 4096 + | 0: 0d 08 14 00 04 00 10 00 0e 05 0a 0f 04 15 00 10 ................ + | 16: 88 02 03 05 90 04 0e 08 00 00 00 00 00 00 00 00 ................ + | 1040: 00 00 00 00 ff 87 7c 02 05 8f 78 0e 08 00 00 00 ......|...x..... + | 2064: 00 00 00 ff 0c 0a 01 fb 00 00 00 00 00 00 00 00 ................ + | 2560: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 83 ................ + | 2576: 78 01 05 87 70 0e 08 00 00 00 00 00 00 00 00 00 x...p........... + | 3072: 00 00 00 00 00 00 00 00 00 ff 00 00 01 fb 00 00 ................ + | 3584: 00 00 00 00 00 83 78 00 05 87 70 0e 08 00 00 00 ......x...p..... + | 4080: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ................ + | end x9.db + SELECT rowid FROM t1; + PRAGMA integrity_check; + +You can run this script to see that the database file is correctly decoded +and loaded. Furthermore, you can make subtle corruptions to the input +database simply by editing the hexadecimal description, then rerun the +script to verify that SQLite correctly handles the corruption. diff --git a/tool/enlargedb.c b/tool/enlargedb.c new file mode 100644 index 0000000..dab5ef1 --- /dev/null +++ b/tool/enlargedb.c @@ -0,0 +1,68 @@ +/* +** Try to enlarge an SQLite database by appending many unused pages. +** The resulting database will fail PRAGMA integrity_check due to the +** appended unused pages, but it should work otherwise. +** +** Usage: +** +** enlargedb DATABASE N +** +** Adds N blank pages onto the end of DATABASE. N can be decimal +** or hex. The total number of pages after adding must be no greater +** than 4294967297 +*/ +#include <stdio.h> +#include <string.h> +#include <stdlib.h> + +int main(int argc, char **argv){ + char *zEnd; + long long int toAppend; + long long int currentSz; + long long int newSz; + FILE *f; + size_t got; + int pgsz; + char zero = 0; + unsigned char buf[100]; + + if( argc!=3 ) goto usage_error; + toAppend = strtoll(argv[2], &zEnd, 0); + if( zEnd==argv[2] || zEnd[0] ) goto usage_error; + if( toAppend<1 ){ + fprintf(stderr, "N must be at least 1\n"); + exit(1); + } + f = fopen(argv[1], "r+b"); + if( f==0 ){ + fprintf(stderr, "cannot open \"%s\" for reading and writing\n", argv[1]); + exit(1); + } + got = fread(buf, 1, sizeof(buf), f); + if( got!=sizeof(buf) ) goto not_valid_db; + if( strcmp((char*)buf,"SQLite format 3")!=0 ) goto not_valid_db; + pgsz = (buf[16]<<8) + buf[17]; + if( pgsz==1 ) pgsz = 65536; + if( pgsz<512 || pgsz>65536 || (pgsz&(pgsz-1))!=0 ) goto not_valid_db; + currentSz = (buf[28]<<24) + (buf[29]<<16) + (buf[30]<<8) + buf[31]; + newSz = currentSz + toAppend; + if( newSz > 0xffffffff ) newSz = 0xffffffff; + buf[28] = (newSz>>24) & 0xff; + buf[29] = (newSz>>16) & 0xff; + buf[30] = (newSz>>8) & 0xff; + buf[31] = newSz & 0xff; + fseek(f, 28, SEEK_SET); + fwrite(&buf[28],4,1,f); + fseek(f, (long)(newSz*pgsz - 1), SEEK_SET); + fwrite(&zero,1,1,f); + fclose(f); + return 0; + +not_valid_db: + fprintf(stderr,"not a valid database: %s\n", argv[1]); + exit(1); + +usage_error: + fprintf(stderr,"Usage: %s DATABASE N\n", argv[0]); + exit(1); +} diff --git a/tool/extract-sqlite3h.tcl b/tool/extract-sqlite3h.tcl new file mode 100755 index 0000000..a0f7c4e --- /dev/null +++ b/tool/extract-sqlite3h.tcl @@ -0,0 +1,21 @@ +#!/usr/bin/tclsh +# +# Given an sqlite3.c source file identified by the command-line +# argument, extract the "sqlite3.h" header file that is embedded inside +# the sqlite3.c source file and write it to standard output. +# +if {[llength $argv]!=1} { + puts stderr "Usage: $argv0 sqlite3.c >sqlite3.h" + exit 1 +} +set in [open [lindex $argv 0] rb] +while {![eof $in]} { + set line [gets $in] + if {[string match {* Begin file sqlite3.h *} $line]} break +} +while {![eof $in]} { + set line [gets $in] + if {[string match {* End of sqlite3.h *} $line]} break + puts $line +} +close $in diff --git a/tool/extract.c b/tool/extract.c new file mode 100644 index 0000000..5bf5caa --- /dev/null +++ b/tool/extract.c @@ -0,0 +1,46 @@ +/* +** Extract a range of bytes from a file. +** +** Usage: +** +** extract FILENAME OFFSET AMOUNT +** +** The bytes are written to standard output. +*/ +#include <stdio.h> +#include <stdlib.h> + +int main(int argc, char **argv){ + FILE *f; + char *zBuf; + int ofst; + int n; + size_t got; + + if( argc!=4 ){ + fprintf(stderr, "Usage: %s FILENAME OFFSET AMOUNT\n", *argv); + return 1; + } + f = fopen(argv[1], "rb"); + if( f==0 ){ + fprintf(stderr, "cannot open \"%s\"\n", argv[1]); + return 1; + } + ofst = atoi(argv[2]); + n = atoi(argv[3]); + zBuf = malloc( n ); + if( zBuf==0 ){ + fprintf(stderr, "out of memory\n"); + return 1; + } + fseek(f, ofst, SEEK_SET); + got = fread(zBuf, 1, n, f); + fclose(f); + if( got<n ){ + fprintf(stderr, "got only %d of %d bytes\n", got, n); + return 1; + }else{ + fwrite(zBuf, 1, n, stdout); + } + return 0; +} diff --git a/tool/fast_vacuum.c b/tool/fast_vacuum.c new file mode 100644 index 0000000..5ca0271 --- /dev/null +++ b/tool/fast_vacuum.c @@ -0,0 +1,234 @@ +/* +** 2013-10-01 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This program implements a high-speed version of the VACUUM command. +** It repacks an SQLite database to remove as much unused space as +** possible and to relocate content sequentially in the file. +** +** This program runs faster and uses less temporary disk space than the +** built-in VACUUM command. On the other hand, this program has a number +** of important restrictions relative to the built-in VACUUM command. +** +** (1) The caller must ensure that no other processes are accessing the +** database file while the vacuum is taking place. The usual SQLite +** file locking is insufficient for this. The caller must use +** external means to make sure only this one routine is reading and +** writing the database. +** +** (2) Database reconfiguration such as page size or auto_vacuum changes +** are not supported by this utility. +** +** (3) The database file might be renamed if a power loss or crash +** occurs at just the wrong moment. Recovery must be prepared to +** to deal with the possibly changed filename. +** +** This program is intended as a *Demonstration Only*. The intent of this +** program is to provide example code that application developers can use +** when creating similar functionality in their applications. +** +** To compile this program: +** +** cc fast_vacuum.c sqlite3.c +** +** Add whatever linker options are required. (Example: "-ldl -lpthread"). +** Then to run the program: +** +** ./a.out file-to-vacuum +** +*/ +#include "sqlite3.h" +#include <stdio.h> +#include <stdlib.h> + +/* +** Finalize a prepared statement. If an error has occurred, print the +** error message and exit. +*/ +static void vacuumFinalize(sqlite3_stmt *pStmt){ + sqlite3 *db = sqlite3_db_handle(pStmt); + int rc = sqlite3_finalize(pStmt); + if( rc ){ + fprintf(stderr, "finalize error: %s\n", sqlite3_errmsg(db)); + exit(1); + } +} + +/* +** Execute zSql on database db. The SQL text is printed to standard +** output. If an error occurs, print an error message and exit the +** process. +*/ +static void execSql(sqlite3 *db, const char *zSql){ + sqlite3_stmt *pStmt; + if( !zSql ){ + fprintf(stderr, "out of memory!\n"); + exit(1); + } + printf("%s;\n", zSql); + if( SQLITE_OK!=sqlite3_prepare(db, zSql, -1, &pStmt, 0) ){ + fprintf(stderr, "Error: %s\n", sqlite3_errmsg(db)); + exit(1); + } + sqlite3_step(pStmt); + vacuumFinalize(pStmt); +} + +/* +** Execute zSql on database db. The zSql statement returns exactly +** one column. Execute this return value as SQL on the same database. +** +** The zSql statement is printed on standard output prior to being +** run. If any errors occur, an error is printed and the process +** exits. +*/ +static void execExecSql(sqlite3 *db, const char *zSql){ + sqlite3_stmt *pStmt; + int rc; + + printf("%s;\n", zSql); + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + fprintf(stderr, "Error: %s\n", sqlite3_errmsg(db)); + exit(1); + } + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + execSql(db, (char*)sqlite3_column_text(pStmt, 0)); + } + vacuumFinalize(pStmt); +} + + +int main(int argc, char **argv){ + sqlite3 *db; /* Connection to the database file */ + int rc; /* Return code from SQLite interface calls */ + sqlite3_uint64 r; /* A random number */ + const char *zDbToVacuum; /* Database to be vacuumed */ + char *zBackupDb; /* Backup copy of the original database */ + char *zTempDb; /* Temporary database */ + char *zSql; /* An SQL statement */ + + if( argc!=2 ){ + fprintf(stderr, "Usage: %s DATABASE\n", argv[0]); + return 1; + } + + /* Identify the database file to be vacuumed and open it. + */ + zDbToVacuum = argv[1]; + printf("-- open database file \"%s\"\n", zDbToVacuum); + rc = sqlite3_open(zDbToVacuum, &db); + if( rc ){ + fprintf(stderr, "%s: %s\n", zDbToVacuum, sqlite3_errstr(rc)); + return 1; + } + + /* Create names for two other files. zTempDb will be a new database + ** into which we construct a vacuumed copy of zDbToVacuum. zBackupDb + ** will be a new name for zDbToVacuum after it is vacuumed. + */ + sqlite3_randomness(sizeof(r), &r); + zTempDb = sqlite3_mprintf("%s-vacuum-%016llx", zDbToVacuum, r); + zBackupDb = sqlite3_mprintf("%s-backup-%016llx", zDbToVacuum, r); + + /* Attach the zTempDb database to the database connection. + */ + zSql = sqlite3_mprintf("ATTACH '%q' AS vacuum_db;", zTempDb); + execSql(db, zSql); + sqlite3_free(zSql); + + /* TODO: + ** Set the page_size and auto_vacuum mode for zTempDb here, if desired. + */ + + /* The vacuum will occur inside of a transaction. Set writable_schema + ** to ON so that we can directly update the sqlite_schema table in the + ** zTempDb database. + */ + execSql(db, "PRAGMA writable_schema=ON"); + execSql(db, "BEGIN"); + + + /* Query the schema of the main database. Create a mirror schema + ** in the temporary database. + */ + execExecSql(db, + "SELECT 'CREATE TABLE vacuum_db.' || substr(sql,14) " + " FROM sqlite_schema WHERE type='table' AND name!='sqlite_sequence'" + " AND rootpage>0" + ); + execExecSql(db, + "SELECT 'CREATE INDEX vacuum_db.' || substr(sql,14)" + " FROM sqlite_schema WHERE sql LIKE 'CREATE INDEX %'" + ); + execExecSql(db, + "SELECT 'CREATE UNIQUE INDEX vacuum_db.' || substr(sql,21) " + " FROM sqlite_schema WHERE sql LIKE 'CREATE UNIQUE INDEX %'" + ); + + /* Loop through the tables in the main database. For each, do + ** an "INSERT INTO vacuum_db.xxx SELECT * FROM main.xxx;" to copy + ** the contents to the temporary database. + */ + execExecSql(db, + "SELECT 'INSERT INTO vacuum_db.' || quote(name) " + "|| ' SELECT * FROM main.' || quote(name) " + "FROM main.sqlite_schema " + "WHERE type = 'table' AND name!='sqlite_sequence' " + " AND rootpage>0" + ); + + /* Copy over the sequence table + */ + execExecSql(db, + "SELECT 'DELETE FROM vacuum_db.' || quote(name) " + "FROM vacuum_db.sqlite_schema WHERE name='sqlite_sequence'" + ); + execExecSql(db, + "SELECT 'INSERT INTO vacuum_db.' || quote(name) " + "|| ' SELECT * FROM main.' || quote(name) " + "FROM vacuum_db.sqlite_schema WHERE name=='sqlite_sequence'" + ); + + /* Copy the triggers, views, and virtual tables from the main database + ** over to the temporary database. None of these objects has any + ** associated storage, so all we have to do is copy their entries + ** from the SQLITE_MASTER table. + */ + execSql(db, + "INSERT INTO vacuum_db.sqlite_schema " + " SELECT type, name, tbl_name, rootpage, sql" + " FROM main.sqlite_schema" + " WHERE type='view' OR type='trigger'" + " OR (type='table' AND rootpage=0)" + ); + + /* Commit the transaction and close the database + */ + execSql(db, "COMMIT"); + printf("-- close database\n"); + sqlite3_close(db); + + + /* At this point, zDbToVacuum is unchanged. zTempDb contains a + ** vacuumed copy of zDbToVacuum. Rearrange filenames so that + ** zTempDb becomes thenew zDbToVacuum. + */ + printf("-- rename \"%s\" to \"%s\"\n", zDbToVacuum, zBackupDb); + rename(zDbToVacuum, zBackupDb); + printf("-- rename \"%s\" to \"%s\"\n", zTempDb, zDbToVacuum); + rename(zTempDb, zDbToVacuum); + + /* Release allocated memory */ + sqlite3_free(zTempDb); + sqlite3_free(zBackupDb); + return 0; +} diff --git a/tool/fragck.tcl b/tool/fragck.tcl new file mode 100644 index 0000000..35e76f4 --- /dev/null +++ b/tool/fragck.tcl @@ -0,0 +1,149 @@ +# Run this TCL script using "testfixture" to get a report that shows +# the sequence of database pages used by a particular table or index. +# This information is used for fragmentation analysis. +# + +# Get the name of the database to analyze +# + +if {[llength $argv]!=2} { + puts stderr "Usage: $argv0 database-name table-or-index-name" + exit 1 +} +set file_to_analyze [lindex $argv 0] +if {![file exists $file_to_analyze]} { + puts stderr "No such file: $file_to_analyze" + exit 1 +} +if {![file readable $file_to_analyze]} { + puts stderr "File is not readable: $file_to_analyze" + exit 1 +} +if {[file size $file_to_analyze]<512} { + puts stderr "Empty or malformed database: $file_to_analyze" + exit 1 +} +set objname [lindex $argv 1] + +# Open the database +# +sqlite3 db [lindex $argv 0] +set DB [btree_open [lindex $argv 0] 1000 0] + +# This proc is a wrapper around the btree_cursor_info command. The +# second argument is an open btree cursor returned by [btree_cursor]. +# The first argument is the name of an array variable that exists in +# the scope of the caller. If the third argument is non-zero, then +# info is returned for the page that lies $up entries upwards in the +# tree-structure. (i.e. $up==1 returns the parent page, $up==2 the +# grandparent etc.) +# +# The following entries in that array are filled in with information retrieved +# using [btree_cursor_info]: +# +# $arrayvar(page_no) = The page number +# $arrayvar(entry_no) = The entry number +# $arrayvar(page_entries) = Total number of entries on this page +# $arrayvar(cell_size) = Cell size (local payload + header) +# $arrayvar(page_freebytes) = Number of free bytes on this page +# $arrayvar(page_freeblocks) = Number of free blocks on the page +# $arrayvar(payload_bytes) = Total payload size (local + overflow) +# $arrayvar(header_bytes) = Header size in bytes +# $arrayvar(local_payload_bytes) = Local payload size +# $arrayvar(parent) = Parent page number +# +proc cursor_info {arrayvar csr {up 0}} { + upvar $arrayvar a + foreach [list a(page_no) \ + a(entry_no) \ + a(page_entries) \ + a(cell_size) \ + a(page_freebytes) \ + a(page_freeblocks) \ + a(payload_bytes) \ + a(header_bytes) \ + a(local_payload_bytes) \ + a(parent) \ + a(first_ovfl) ] [btree_cursor_info $csr $up] break +} + +# Determine the page-size of the database. This global variable is used +# throughout the script. +# +set pageSize [db eval {PRAGMA page_size}] + +# Find the root page of table or index to be analyzed. Also find out +# if the object is a table or an index. +# +if {$objname=="sqlite_master"} { + set rootpage 1 + set type table +} else { + db eval { + SELECT rootpage, type FROM sqlite_master + WHERE name=$objname + } break + if {![info exists rootpage]} { + puts stderr "no such table or index: $objname" + exit 1 + } + if {$type!="table" && $type!="index"} { + puts stderr "$objname is something other than a table or index" + exit 1 + } + if {![string is integer -strict $rootpage]} { + puts stderr "invalid root page for $objname: $rootpage" + exit 1 + } +} + +# The cursor $csr is pointing to an entry. Print out information +# about the page that $up levels above that page that contains +# the entry. If $up==0 use the page that contains the entry. +# +# If information about the page has been printed already, then +# this is a no-op. +# +proc page_info {csr up} { + global seen + cursor_info ci $csr $up + set pg $ci(page_no) + if {[info exists seen($pg)]} return + set seen($pg) 1 + + # Do parent pages first + # + if {$ci(parent)} { + page_info $csr [expr {$up+1}] + } + + # Find the depth of this page + # + set depth 1 + set i $up + while {$ci(parent)} { + incr i + incr depth + cursor_info ci $csr $i + } + + # print the results + # + puts [format {LEVEL %d: %6d} $depth $pg] +} + + + + +# Loop through the object and print out page numbers +# +set csr [btree_cursor $DB $rootpage 0] +for {btree_first $csr} {![btree_eof $csr]} {btree_next $csr} { + page_info $csr 0 + set i 1 + foreach pg [btree_ovfl_info $DB $csr] { + puts [format {OVFL %3d: %6d} $i $pg] + incr i + } +} +exit 0 diff --git a/tool/fuzzershell.c b/tool/fuzzershell.c new file mode 100644 index 0000000..7a7aef0 --- /dev/null +++ b/tool/fuzzershell.c @@ -0,0 +1,1267 @@ +/* +** 2015-04-17 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This is a utility program designed to aid running the SQLite library +** against an external fuzzer, such as American Fuzzy Lop (AFL) +** (http://lcamtuf.coredump.cx/afl/). Basically, this program reads +** SQL text from standard input and passes it through to SQLite for evaluation, +** just like the "sqlite3" command-line shell. Differences from the +** command-line shell: +** +** (1) The complex "dot-command" extensions are omitted. This +** prevents the fuzzer from discovering that it can run things +** like ".shell rm -rf ~" +** +** (2) The database is opened with the SQLITE_OPEN_MEMORY flag so that +** no disk I/O from the database is permitted. The ATTACH command +** with a filename still uses an in-memory database. +** +** (3) The main in-memory database can be initialized from a template +** disk database so that the fuzzer starts with a database containing +** content. +** +** (4) The eval() SQL function is added, allowing the fuzzer to do +** interesting recursive operations. +** +** (5) An error is raised if there is a memory leak. +** +** The input text can be divided into separate test cases using comments +** of the form: +** +** |****<...>****| +** +** where the "..." is arbitrary text. (Except the "|" should really be "/". +** "|" is used here to avoid compiler errors about nested comments.) +** A separate in-memory SQLite database is created to run each test case. +** This feature allows the "queue" of AFL to be captured into a single big +** file using a command like this: +** +** (for i in id:*; do echo '|****<'$i'>****|'; cat $i; done) >~/all-queue.txt +** +** (Once again, change the "|" to "/") Then all elements of the AFL queue +** can be run in a single go (for regression testing, for example) by typing: +** +** fuzzershell -f ~/all-queue.txt +** +** After running each chunk of SQL, the database connection is closed. The +** program aborts if the close fails or if there is any unfreed memory after +** the close. +** +** New test cases can be appended to all-queue.txt at any time. If redundant +** test cases are added, they can be eliminated by running: +** +** fuzzershell -f ~/all-queue.txt --unique-cases ~/unique-cases.txt +*/ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <stdarg.h> +#include <ctype.h> +#include "sqlite3.h" +#define ISDIGIT(X) isdigit((unsigned char)(X)) + +/* +** All global variables are gathered into the "g" singleton. +*/ +struct GlobalVars { + const char *zArgv0; /* Name of program */ + sqlite3_mem_methods sOrigMem; /* Original memory methods */ + sqlite3_mem_methods sOomMem; /* Memory methods with OOM simulator */ + int iOomCntdown; /* Memory fails on 1 to 0 transition */ + int nOomFault; /* Increments for each OOM fault */ + int bOomOnce; /* Fail just once if true */ + int bOomEnable; /* True to enable OOM simulation */ + int nOomBrkpt; /* Number of calls to oomFault() */ + char zTestName[100]; /* Name of current test */ +} g; + +/* +** Maximum number of iterations for an OOM test +*/ +#ifndef OOM_MAX +# define OOM_MAX 625 +#endif + +/* +** This routine is called when a simulated OOM occurs. It exists as a +** convenient place to set a debugger breakpoint. +*/ +static void oomFault(void){ + g.nOomBrkpt++; /* Prevent oomFault() from being optimized out */ +} + + +/* Versions of malloc() and realloc() that simulate OOM conditions */ +static void *oomMalloc(int nByte){ + if( nByte>0 && g.bOomEnable && g.iOomCntdown>0 ){ + g.iOomCntdown--; + if( g.iOomCntdown==0 ){ + if( g.nOomFault==0 ) oomFault(); + g.nOomFault++; + if( !g.bOomOnce ) g.iOomCntdown = 1; + return 0; + } + } + return g.sOrigMem.xMalloc(nByte); +} +static void *oomRealloc(void *pOld, int nByte){ + if( nByte>0 && g.bOomEnable && g.iOomCntdown>0 ){ + g.iOomCntdown--; + if( g.iOomCntdown==0 ){ + if( g.nOomFault==0 ) oomFault(); + g.nOomFault++; + if( !g.bOomOnce ) g.iOomCntdown = 1; + return 0; + } + } + return g.sOrigMem.xRealloc(pOld, nByte); +} + +/* +** Print an error message and abort in such a way to indicate to the +** fuzzer that this counts as a crash. +*/ +static void abendError(const char *zFormat, ...){ + va_list ap; + if( g.zTestName[0] ){ + fprintf(stderr, "%s (%s): ", g.zArgv0, g.zTestName); + }else{ + fprintf(stderr, "%s: ", g.zArgv0); + } + va_start(ap, zFormat); + vfprintf(stderr, zFormat, ap); + va_end(ap); + fprintf(stderr, "\n"); + abort(); +} +/* +** Print an error message and quit, but not in a way that would look +** like a crash. +*/ +static void fatalError(const char *zFormat, ...){ + va_list ap; + if( g.zTestName[0] ){ + fprintf(stderr, "%s (%s): ", g.zArgv0, g.zTestName); + }else{ + fprintf(stderr, "%s: ", g.zArgv0); + } + va_start(ap, zFormat); + vfprintf(stderr, zFormat, ap); + va_end(ap); + fprintf(stderr, "\n"); + exit(1); +} + +/* +** Evaluate some SQL. Abort if unable. +*/ +static void sqlexec(sqlite3 *db, const char *zFormat, ...){ + va_list ap; + char *zSql; + char *zErrMsg = 0; + int rc; + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + rc = sqlite3_exec(db, zSql, 0, 0, &zErrMsg); + if( rc ) abendError("failed sql [%s]: %s", zSql, zErrMsg); + sqlite3_free(zSql); +} + +/* +** This callback is invoked by sqlite3_log(). +*/ +static void shellLog(void *pNotUsed, int iErrCode, const char *zMsg){ + printf("LOG: (%d) %s\n", iErrCode, zMsg); + fflush(stdout); +} +static void shellLogNoop(void *pNotUsed, int iErrCode, const char *zMsg){ + return; +} + +/* +** This callback is invoked by sqlite3_exec() to return query results. +*/ +static int execCallback(void *NotUsed, int argc, char **argv, char **colv){ + int i; + static unsigned cnt = 0; + printf("ROW #%u:\n", ++cnt); + if( argv ){ + for(i=0; i<argc; i++){ + printf(" %s=", colv[i]); + if( argv[i] ){ + printf("[%s]\n", argv[i]); + }else{ + printf("NULL\n"); + } + } + } + fflush(stdout); + return 0; +} +static int execNoop(void *NotUsed, int argc, char **argv, char **colv){ + return 0; +} + +#ifndef SQLITE_OMIT_TRACE +/* +** This callback is invoked by sqlite3_trace() as each SQL statement +** starts. +*/ +static void traceCallback(void *NotUsed, const char *zMsg){ + printf("TRACE: %s\n", zMsg); + fflush(stdout); +} +static void traceNoop(void *NotUsed, const char *zMsg){ + return; +} +#endif + +/*************************************************************************** +** String accumulator object +*/ +typedef struct Str Str; +struct Str { + char *z; /* The string. Memory from malloc() */ + sqlite3_uint64 n; /* Bytes of input used */ + sqlite3_uint64 nAlloc; /* Bytes allocated to z[] */ + int oomErr; /* OOM error has been seen */ +}; + +/* Initialize a Str object */ +static void StrInit(Str *p){ + memset(p, 0, sizeof(*p)); +} + +/* Append text to the end of a Str object */ +static void StrAppend(Str *p, const char *z){ + sqlite3_uint64 n = strlen(z); + if( p->n + n >= p->nAlloc ){ + char *zNew; + sqlite3_uint64 nNew; + if( p->oomErr ) return; + nNew = p->nAlloc*2 + 100 + n; + zNew = sqlite3_realloc(p->z, (int)nNew); + if( zNew==0 ){ + sqlite3_free(p->z); + memset(p, 0, sizeof(*p)); + p->oomErr = 1; + return; + } + p->z = zNew; + p->nAlloc = nNew; + } + memcpy(p->z + p->n, z, (size_t)n); + p->n += n; + p->z[p->n] = 0; +} + +/* Return the current string content */ +static char *StrStr(Str *p){ + return p->z; +} + +/* Free the string */ +static void StrFree(Str *p){ + sqlite3_free(p->z); + StrInit(p); +} + +/*************************************************************************** +** eval() implementation copied from ../ext/misc/eval.c +*/ +/* +** Structure used to accumulate the output +*/ +struct EvalResult { + char *z; /* Accumulated output */ + const char *zSep; /* Separator */ + int szSep; /* Size of the separator string */ + sqlite3_int64 nAlloc; /* Number of bytes allocated for z[] */ + sqlite3_int64 nUsed; /* Number of bytes of z[] actually used */ +}; + +/* +** Callback from sqlite_exec() for the eval() function. +*/ +static int callback(void *pCtx, int argc, char **argv, char **colnames){ + struct EvalResult *p = (struct EvalResult*)pCtx; + int i; + for(i=0; i<argc; i++){ + const char *z = argv[i] ? argv[i] : ""; + size_t sz = strlen(z); + if( (sqlite3_int64)sz+p->nUsed+p->szSep+1 > p->nAlloc ){ + char *zNew; + p->nAlloc = p->nAlloc*2 + sz + p->szSep + 1; + /* Using sqlite3_realloc64() would be better, but it is a recent + ** addition and will cause a segfault if loaded by an older version + ** of SQLite. */ + zNew = p->nAlloc<=0x7fffffff ? sqlite3_realloc(p->z, (int)p->nAlloc) : 0; + if( zNew==0 ){ + sqlite3_free(p->z); + memset(p, 0, sizeof(*p)); + return 1; + } + p->z = zNew; + } + if( p->nUsed>0 ){ + memcpy(&p->z[p->nUsed], p->zSep, p->szSep); + p->nUsed += p->szSep; + } + memcpy(&p->z[p->nUsed], z, sz); + p->nUsed += sz; + } + return 0; +} + +/* +** Implementation of the eval(X) and eval(X,Y) SQL functions. +** +** Evaluate the SQL text in X. Return the results, using string +** Y as the separator. If Y is omitted, use a single space character. +*/ +static void sqlEvalFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zSql; + sqlite3 *db; + char *zErr = 0; + int rc; + struct EvalResult x; + + memset(&x, 0, sizeof(x)); + x.zSep = " "; + zSql = (const char*)sqlite3_value_text(argv[0]); + if( zSql==0 ) return; + if( argc>1 ){ + x.zSep = (const char*)sqlite3_value_text(argv[1]); + if( x.zSep==0 ) return; + } + x.szSep = (int)strlen(x.zSep); + db = sqlite3_context_db_handle(context); + rc = sqlite3_exec(db, zSql, callback, &x, &zErr); + if( rc!=SQLITE_OK ){ + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); + }else if( x.zSep==0 ){ + sqlite3_result_error_nomem(context); + sqlite3_free(x.z); + }else{ + sqlite3_result_text(context, x.z, (int)x.nUsed, sqlite3_free); + } +} +/* End of the eval() implementation +******************************************************************************/ + +/****************************************************************************** +** The generate_series(START,END,STEP) eponymous table-valued function. +** +** This code is copy/pasted from ext/misc/series.c in the SQLite source tree. +*/ +/* series_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result +*/ +typedef struct series_cursor series_cursor; +struct series_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + int isDesc; /* True to count down rather than up */ + sqlite3_int64 iRowid; /* The rowid */ + sqlite3_int64 iValue; /* Current value ("value") */ + sqlite3_int64 mnValue; /* Mimimum value ("start") */ + sqlite3_int64 mxValue; /* Maximum value ("stop") */ + sqlite3_int64 iStep; /* Increment ("step") */ +}; + +/* +** The seriesConnect() method is invoked to create a new +** series_vtab that describes the generate_series virtual table. +** +** Think of this routine as the constructor for series_vtab objects. +** +** All this routine needs to do is: +** +** (1) Allocate the series_vtab object and initialize all fields. +** +** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the +** result set of queries against generate_series will look like. +*/ +static int seriesConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + sqlite3_vtab *pNew; + int rc; + +/* Column numbers */ +#define SERIES_COLUMN_VALUE 0 +#define SERIES_COLUMN_START 1 +#define SERIES_COLUMN_STOP 2 +#define SERIES_COLUMN_STEP 3 + + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(value,start hidden,stop hidden,step hidden)"); + if( rc==SQLITE_OK ){ + pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + } + return rc; +} + +/* +** This method is the destructor for series_cursor objects. +*/ +static int seriesDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new series_cursor object. +*/ +static int seriesOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + series_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Destructor for a series_cursor. +*/ +static int seriesClose(sqlite3_vtab_cursor *cur){ + sqlite3_free(cur); + return SQLITE_OK; +} + + +/* +** Advance a series_cursor to its next row of output. +*/ +static int seriesNext(sqlite3_vtab_cursor *cur){ + series_cursor *pCur = (series_cursor*)cur; + if( pCur->isDesc ){ + pCur->iValue -= pCur->iStep; + }else{ + pCur->iValue += pCur->iStep; + } + pCur->iRowid++; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the series_cursor +** is currently pointing. +*/ +static int seriesColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + series_cursor *pCur = (series_cursor*)cur; + sqlite3_int64 x = 0; + switch( i ){ + case SERIES_COLUMN_START: x = pCur->mnValue; break; + case SERIES_COLUMN_STOP: x = pCur->mxValue; break; + case SERIES_COLUMN_STEP: x = pCur->iStep; break; + default: x = pCur->iValue; break; + } + sqlite3_result_int64(ctx, x); + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** rowid is the same as the output value. +*/ +static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + series_cursor *pCur = (series_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int seriesEof(sqlite3_vtab_cursor *cur){ + series_cursor *pCur = (series_cursor*)cur; + if( pCur->isDesc ){ + return pCur->iValue < pCur->mnValue; + }else{ + return pCur->iValue > pCur->mxValue; + } +} + +/* True to cause run-time checking of the start=, stop=, and/or step= +** parameters. The only reason to do this is for testing the +** constraint checking logic for virtual tables in the SQLite core. +*/ +#ifndef SQLITE_SERIES_CONSTRAINT_VERIFY +# define SQLITE_SERIES_CONSTRAINT_VERIFY 0 +#endif + +/* +** This method is called to "rewind" the series_cursor object back +** to the first row of output. This method is always called at least +** once prior to any call to seriesColumn() or seriesRowid() or +** seriesEof(). +** +** The query plan selected by seriesBestIndex is passed in the idxNum +** parameter. (idxStr is not used in this implementation.) idxNum +** is a bitmask showing which constraints are available: +** +** 1: start=VALUE +** 2: stop=VALUE +** 4: step=VALUE +** +** Also, if bit 8 is set, that means that the series should be output +** in descending order rather than in ascending order. +** +** This routine should initialize the cursor and position it so that it +** is pointing at the first row, or pointing off the end of the table +** (so that seriesEof() will return true) if the table is empty. +*/ +static int seriesFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + series_cursor *pCur = (series_cursor *)pVtabCursor; + int i = 0; + if( idxNum & 1 ){ + pCur->mnValue = sqlite3_value_int64(argv[i++]); + }else{ + pCur->mnValue = 0; + } + if( idxNum & 2 ){ + pCur->mxValue = sqlite3_value_int64(argv[i++]); + }else{ + pCur->mxValue = 0xffffffff; + } + if( idxNum & 4 ){ + pCur->iStep = sqlite3_value_int64(argv[i++]); + if( pCur->iStep<1 ) pCur->iStep = 1; + }else{ + pCur->iStep = 1; + } + if( idxNum & 8 ){ + pCur->isDesc = 1; + pCur->iValue = pCur->mxValue; + if( pCur->iStep>0 ){ + pCur->iValue -= (pCur->mxValue - pCur->mnValue)%pCur->iStep; + } + }else{ + pCur->isDesc = 0; + pCur->iValue = pCur->mnValue; + } + pCur->iRowid = 1; + return SQLITE_OK; +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the generate_series virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +** +** In this implementation idxNum is used to represent the +** query plan. idxStr is unused. +** +** The query plan is represented by bits in idxNum: +** +** (1) start = $value -- constraint exists +** (2) stop = $value -- constraint exists +** (4) step = $value -- constraint exists +** (8) output in descending order +*/ +static int seriesBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; /* Loop over constraints */ + int idxNum = 0; /* The query plan bitmask */ + int startIdx = -1; /* Index of the start= constraint, or -1 if none */ + int stopIdx = -1; /* Index of the stop= constraint, or -1 if none */ + int stepIdx = -1; /* Index of the step= constraint, or -1 if none */ + int nArg = 0; /* Number of arguments that seriesFilter() expects */ + + const struct sqlite3_index_constraint *pConstraint; + pConstraint = pIdxInfo->aConstraint; + for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + switch( pConstraint->iColumn ){ + case SERIES_COLUMN_START: + startIdx = i; + idxNum |= 1; + break; + case SERIES_COLUMN_STOP: + stopIdx = i; + idxNum |= 2; + break; + case SERIES_COLUMN_STEP: + stepIdx = i; + idxNum |= 4; + break; + } + } + if( startIdx>=0 ){ + pIdxInfo->aConstraintUsage[startIdx].argvIndex = ++nArg; + pIdxInfo->aConstraintUsage[startIdx].omit= !SQLITE_SERIES_CONSTRAINT_VERIFY; + } + if( stopIdx>=0 ){ + pIdxInfo->aConstraintUsage[stopIdx].argvIndex = ++nArg; + pIdxInfo->aConstraintUsage[stopIdx].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY; + } + if( stepIdx>=0 ){ + pIdxInfo->aConstraintUsage[stepIdx].argvIndex = ++nArg; + pIdxInfo->aConstraintUsage[stepIdx].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY; + } + if( (idxNum & 3)==3 ){ + /* Both start= and stop= boundaries are available. This is the + ** the preferred case */ + pIdxInfo->estimatedCost = (double)(2 - ((idxNum&4)!=0)); + pIdxInfo->estimatedRows = 1000; + if( pIdxInfo->nOrderBy==1 ){ + if( pIdxInfo->aOrderBy[0].desc ) idxNum |= 8; + pIdxInfo->orderByConsumed = 1; + } + }else{ + /* If either boundary is missing, we have to generate a huge span + ** of numbers. Make this case very expensive so that the query + ** planner will work hard to avoid it. */ + pIdxInfo->estimatedCost = (double)2147483647; + pIdxInfo->estimatedRows = 2147483647; + } + pIdxInfo->idxNum = idxNum; + return SQLITE_OK; +} + +/* +** This following structure defines all the methods for the +** generate_series virtual table. +*/ +static sqlite3_module seriesModule = { + 0, /* iVersion */ + 0, /* xCreate */ + seriesConnect, /* xConnect */ + seriesBestIndex, /* xBestIndex */ + seriesDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + seriesOpen, /* xOpen - open a cursor */ + seriesClose, /* xClose - close a cursor */ + seriesFilter, /* xFilter - configure scan constraints */ + seriesNext, /* xNext - advance a cursor */ + seriesEof, /* xEof - check for end of scan */ + seriesColumn, /* xColumn - read data */ + seriesRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ +}; +/* END the generate_series(START,END,STEP) implementation +*********************************************************************************/ + +/* +** Print sketchy documentation for this utility program +*/ +static void showHelp(void){ + printf("Usage: %s [options] ?FILE...?\n", g.zArgv0); + printf( +"Read SQL text from FILE... (or from standard input if FILE... is omitted)\n" +"and then evaluate each block of SQL contained therein.\n" +"Options:\n" +" --autovacuum Enable AUTOVACUUM mode\n" +" --database FILE Use database FILE instead of an in-memory database\n" +" --disable-lookaside Turn off lookaside memory\n" +" --heap SZ MIN Memory allocator uses SZ bytes & min allocation MIN\n" +" --help Show this help text\n" +" --lookaside N SZ Configure lookaside for N slots of SZ bytes each\n" +" --oom Run each test multiple times in a simulated OOM loop\n" +" --pagesize N Set the page size to N\n" +" --pcache N SZ Configure N pages of pagecache each of size SZ bytes\n" +" -q Reduced output\n" +" --quiet Reduced output\n" +" --scratch N SZ Configure scratch memory for N slots of SZ bytes each\n" +" --unique-cases FILE Write all unique test cases to FILE\n" +" --utf16be Set text encoding to UTF-16BE\n" +" --utf16le Set text encoding to UTF-16LE\n" +" -v Increased output\n" +" --verbose Increased output\n" + ); +} + +/* +** Return the value of a hexadecimal digit. Return -1 if the input +** is not a hex digit. +*/ +static int hexDigitValue(char c){ + if( c>='0' && c<='9' ) return c - '0'; + if( c>='a' && c<='f' ) return c - 'a' + 10; + if( c>='A' && c<='F' ) return c - 'A' + 10; + return -1; +} + +/* +** Interpret zArg as an integer value, possibly with suffixes. +*/ +static int integerValue(const char *zArg){ + sqlite3_int64 v = 0; + static const struct { char *zSuffix; int iMult; } aMult[] = { + { "KiB", 1024 }, + { "MiB", 1024*1024 }, + { "GiB", 1024*1024*1024 }, + { "KB", 1000 }, + { "MB", 1000000 }, + { "GB", 1000000000 }, + { "K", 1000 }, + { "M", 1000000 }, + { "G", 1000000000 }, + }; + int i; + int isNeg = 0; + if( zArg[0]=='-' ){ + isNeg = 1; + zArg++; + }else if( zArg[0]=='+' ){ + zArg++; + } + if( zArg[0]=='0' && zArg[1]=='x' ){ + int x; + zArg += 2; + while( (x = hexDigitValue(zArg[0]))>=0 ){ + v = (v<<4) + x; + zArg++; + } + }else{ + while( ISDIGIT(zArg[0]) ){ + v = v*10 + zArg[0] - '0'; + zArg++; + } + } + for(i=0; i<sizeof(aMult)/sizeof(aMult[0]); i++){ + if( sqlite3_stricmp(aMult[i].zSuffix, zArg)==0 ){ + v *= aMult[i].iMult; + break; + } + } + if( v>0x7fffffff ) abendError("parameter too large - max 2147483648"); + return (int)(isNeg? -v : v); +} + +/* Return the current wall-clock time */ +static sqlite3_int64 timeOfDay(void){ + static sqlite3_vfs *clockVfs = 0; + sqlite3_int64 t; + if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0); + if( clockVfs->iVersion>=1 && clockVfs->xCurrentTimeInt64!=0 ){ + clockVfs->xCurrentTimeInt64(clockVfs, &t); + }else{ + double r; + clockVfs->xCurrentTime(clockVfs, &r); + t = (sqlite3_int64)(r*86400000.0); + } + return t; +} + +int main(int argc, char **argv){ + char *zIn = 0; /* Input text */ + int nAlloc = 0; /* Number of bytes allocated for zIn[] */ + int nIn = 0; /* Number of bytes of zIn[] used */ + size_t got; /* Bytes read from input */ + int rc = SQLITE_OK; /* Result codes from API functions */ + int i; /* Loop counter */ + int iNext; /* Next block of SQL */ + sqlite3 *db; /* Open database */ + char *zErrMsg = 0; /* Error message returned from sqlite3_exec() */ + const char *zEncoding = 0; /* --utf16be or --utf16le */ + int nHeap = 0, mnHeap = 0; /* Heap size from --heap */ + int nLook = 0, szLook = 0; /* --lookaside configuration */ + int nPCache = 0, szPCache = 0;/* --pcache configuration */ + int nScratch = 0, szScratch=0;/* --scratch configuration */ + int pageSize = 0; /* Desired page size. 0 means default */ + void *pHeap = 0; /* Allocated heap space */ + void *pLook = 0; /* Allocated lookaside space */ + void *pPCache = 0; /* Allocated storage for pcache */ + void *pScratch = 0; /* Allocated storage for scratch */ + int doAutovac = 0; /* True for --autovacuum */ + char *zSql; /* SQL to run */ + char *zToFree = 0; /* Call sqlite3_free() on this afte running zSql */ + int verboseFlag = 0; /* --verbose or -v flag */ + int quietFlag = 0; /* --quiet or -q flag */ + int nTest = 0; /* Number of test cases run */ + int multiTest = 0; /* True if there will be multiple test cases */ + int lastPct = -1; /* Previous percentage done output */ + sqlite3 *dataDb = 0; /* Database holding compacted input data */ + sqlite3_stmt *pStmt = 0; /* Statement to insert testcase into dataDb */ + const char *zDataOut = 0; /* Write compacted data to this output file */ + int nHeader = 0; /* Bytes of header comment text on input file */ + int oomFlag = 0; /* --oom */ + int oomCnt = 0; /* Counter for the OOM loop */ + char zErrBuf[200]; /* Space for the error message */ + const char *zFailCode; /* Value of the TEST_FAILURE environment var */ + const char *zPrompt; /* Initial prompt when large-file fuzzing */ + int nInFile = 0; /* Number of input files to read */ + char **azInFile = 0; /* Array of input file names */ + int jj; /* Loop counter for azInFile[] */ + sqlite3_int64 iBegin; /* Start time for the whole program */ + sqlite3_int64 iStart, iEnd; /* Start and end-times for a test case */ + const char *zDbName = 0; /* Name of an on-disk database file to open */ + + iBegin = timeOfDay(); + sqlite3_shutdown(); + zFailCode = getenv("TEST_FAILURE"); + g.zArgv0 = argv[0]; + zPrompt = "<stdin>"; + for(i=1; i<argc; i++){ + const char *z = argv[i]; + if( z[0]=='-' ){ + z++; + if( z[0]=='-' ) z++; + if( strcmp(z,"autovacuum")==0 ){ + doAutovac = 1; + }else + if( strcmp(z,"database")==0 ){ + if( i>=argc-1 ) abendError("missing argument on %s\n", argv[i]); + zDbName = argv[i+1]; + i += 1; + }else + if( strcmp(z,"disable-lookaside")==0 ){ + nLook = 1; + szLook = 0; + }else + if( strcmp(z, "f")==0 && i+1<argc ){ + i++; + goto addNewInFile; + }else + if( strcmp(z,"heap")==0 ){ + if( i>=argc-2 ) abendError("missing arguments on %s\n", argv[i]); + nHeap = integerValue(argv[i+1]); + mnHeap = integerValue(argv[i+2]); + i += 2; + }else + if( strcmp(z,"help")==0 ){ + showHelp(); + return 0; + }else + if( strcmp(z,"lookaside")==0 ){ + if( i>=argc-2 ) abendError("missing arguments on %s", argv[i]); + nLook = integerValue(argv[i+1]); + szLook = integerValue(argv[i+2]); + i += 2; + }else + if( strcmp(z,"oom")==0 ){ + oomFlag = 1; + }else + if( strcmp(z,"pagesize")==0 ){ + if( i>=argc-1 ) abendError("missing argument on %s", argv[i]); + pageSize = integerValue(argv[++i]); + }else + if( strcmp(z,"pcache")==0 ){ + if( i>=argc-2 ) abendError("missing arguments on %s", argv[i]); + nPCache = integerValue(argv[i+1]); + szPCache = integerValue(argv[i+2]); + i += 2; + }else + if( strcmp(z,"quiet")==0 || strcmp(z,"q")==0 ){ + quietFlag = 1; + verboseFlag = 0; + }else + if( strcmp(z,"scratch")==0 ){ + if( i>=argc-2 ) abendError("missing arguments on %s", argv[i]); + nScratch = integerValue(argv[i+1]); + szScratch = integerValue(argv[i+2]); + i += 2; + }else + if( strcmp(z, "unique-cases")==0 ){ + if( i>=argc-1 ) abendError("missing arguments on %s", argv[i]); + if( zDataOut ) abendError("only one --minimize allowed"); + zDataOut = argv[++i]; + }else + if( strcmp(z,"utf16le")==0 ){ + zEncoding = "utf16le"; + }else + if( strcmp(z,"utf16be")==0 ){ + zEncoding = "utf16be"; + }else + if( strcmp(z,"verbose")==0 || strcmp(z,"v")==0 ){ + quietFlag = 0; + verboseFlag = 1; + }else + { + abendError("unknown option: %s", argv[i]); + } + }else{ + addNewInFile: + nInFile++; + azInFile = realloc(azInFile, sizeof(azInFile[0])*nInFile); + if( azInFile==0 ) abendError("out of memory"); + azInFile[nInFile-1] = argv[i]; + } + } + + /* Do global SQLite initialization */ + sqlite3_config(SQLITE_CONFIG_LOG, verboseFlag ? shellLog : shellLogNoop, 0); + if( nHeap>0 ){ + pHeap = malloc( nHeap ); + if( pHeap==0 ) fatalError("cannot allocate %d-byte heap\n", nHeap); + rc = sqlite3_config(SQLITE_CONFIG_HEAP, pHeap, nHeap, mnHeap); + if( rc ) abendError("heap configuration failed: %d\n", rc); + } + if( oomFlag ){ + sqlite3_config(SQLITE_CONFIG_GETMALLOC, &g.sOrigMem); + g.sOomMem = g.sOrigMem; + g.sOomMem.xMalloc = oomMalloc; + g.sOomMem.xRealloc = oomRealloc; + sqlite3_config(SQLITE_CONFIG_MALLOC, &g.sOomMem); + } + if( nLook>0 ){ + sqlite3_config(SQLITE_CONFIG_LOOKASIDE, 0, 0); + if( szLook>0 ){ + pLook = malloc( nLook*szLook ); + if( pLook==0 ) fatalError("out of memory"); + } + } + if( nScratch>0 && szScratch>0 ){ + pScratch = malloc( nScratch*(sqlite3_int64)szScratch ); + if( pScratch==0 ) fatalError("cannot allocate %lld-byte scratch", + nScratch*(sqlite3_int64)szScratch); + rc = sqlite3_config(SQLITE_CONFIG_SCRATCH, pScratch, szScratch, nScratch); + if( rc ) abendError("scratch configuration failed: %d\n", rc); + } + if( nPCache>0 && szPCache>0 ){ + pPCache = malloc( nPCache*(sqlite3_int64)szPCache ); + if( pPCache==0 ) fatalError("cannot allocate %lld-byte pcache", + nPCache*(sqlite3_int64)szPCache); + rc = sqlite3_config(SQLITE_CONFIG_PAGECACHE, pPCache, szPCache, nPCache); + if( rc ) abendError("pcache configuration failed: %d", rc); + } + + /* If the --unique-cases option was supplied, open the database that will + ** be used to gather unique test cases. + */ + if( zDataOut ){ + rc = sqlite3_open(":memory:", &dataDb); + if( rc ) abendError("cannot open :memory: database"); + rc = sqlite3_exec(dataDb, + "CREATE TABLE testcase(sql BLOB PRIMARY KEY, tm) WITHOUT ROWID;",0,0,0); + if( rc ) abendError("%s", sqlite3_errmsg(dataDb)); + rc = sqlite3_prepare_v2(dataDb, + "INSERT OR IGNORE INTO testcase(sql,tm)VALUES(?1,?2)", + -1, &pStmt, 0); + if( rc ) abendError("%s", sqlite3_errmsg(dataDb)); + } + + /* Initialize the input buffer used to hold SQL text */ + if( nInFile==0 ) nInFile = 1; + nAlloc = 1000; + zIn = malloc(nAlloc); + if( zIn==0 ) fatalError("out of memory"); + + /* Loop over all input files */ + for(jj=0; jj<nInFile; jj++){ + + /* Read the complete content of the next input file into zIn[] */ + FILE *in; + if( azInFile ){ + int j, k; + in = fopen(azInFile[jj],"rb"); + if( in==0 ){ + abendError("cannot open %s for reading", azInFile[jj]); + } + zPrompt = azInFile[jj]; + for(j=k=0; zPrompt[j]; j++) if( zPrompt[j]=='/' ) k = j+1; + zPrompt += k; + }else{ + in = stdin; + zPrompt = "<stdin>"; + } + while( !feof(in) ){ + got = fread(zIn+nIn, 1, nAlloc-nIn-1, in); + nIn += (int)got; + zIn[nIn] = 0; + if( got==0 ) break; + if( nAlloc - nIn - 1 < 100 ){ + nAlloc += nAlloc+1000; + zIn = realloc(zIn, nAlloc); + if( zIn==0 ) fatalError("out of memory"); + } + } + if( in!=stdin ) fclose(in); + lastPct = -1; + + /* Skip initial lines of the input file that begin with "#" */ + for(i=0; i<nIn; i=iNext+1){ + if( zIn[i]!='#' ) break; + for(iNext=i+1; iNext<nIn && zIn[iNext]!='\n'; iNext++){} + } + nHeader = i; + + /* Process all test cases contained within the input file. + */ + for(; i<nIn; i=iNext, nTest++, g.zTestName[0]=0){ + char cSaved; + if( strncmp(&zIn[i], "/****<",6)==0 ){ + char *z = strstr(&zIn[i], ">****/"); + if( z ){ + z += 6; + sqlite3_snprintf(sizeof(g.zTestName), g.zTestName, "%.*s", + (int)(z-&zIn[i]) - 12, &zIn[i+6]); + if( verboseFlag ){ + printf("%.*s\n", (int)(z-&zIn[i]), &zIn[i]); + fflush(stdout); + } + i += (int)(z-&zIn[i]); + multiTest = 1; + } + } + for(iNext=i; iNext<nIn && strncmp(&zIn[iNext],"/****<",6)!=0; iNext++){} + cSaved = zIn[iNext]; + zIn[iNext] = 0; + + + /* Print out the SQL of the next test case is --verbose is enabled + */ + zSql = &zIn[i]; + if( verboseFlag ){ + printf("INPUT (offset: %d, size: %d): [%s]\n", + i, (int)strlen(&zIn[i]), &zIn[i]); + }else if( multiTest && !quietFlag ){ + if( oomFlag ){ + printf("%s\n", g.zTestName); + }else{ + int pct = (10*iNext)/nIn; + if( pct!=lastPct ){ + if( lastPct<0 ) printf("%s:", zPrompt); + printf(" %d%%", pct*10); + lastPct = pct; + } + } + }else if( nInFile>1 ){ + printf("%s\n", zPrompt); + } + fflush(stdout); + + /* Run the next test case. Run it multiple times in --oom mode + */ + if( oomFlag ){ + oomCnt = g.iOomCntdown = 1; + g.nOomFault = 0; + g.bOomOnce = 1; + if( verboseFlag ){ + printf("Once.%d\n", oomCnt); + fflush(stdout); + } + }else{ + oomCnt = 0; + } + do{ + Str sql; + StrInit(&sql); + if( zDbName ){ + rc = sqlite3_open_v2(zDbName, &db, SQLITE_OPEN_READWRITE, 0); + if( rc!=SQLITE_OK ){ + abendError("Cannot open database file %s", zDbName); + } + }else{ + rc = sqlite3_open_v2( + "main.db", &db, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY, + 0); + if( rc!=SQLITE_OK ){ + abendError("Unable to open the in-memory database"); + } + } + if( pLook ){ + rc = sqlite3_db_config(db, SQLITE_DBCONFIG_LOOKASIDE,pLook,szLook,nLook); + if( rc!=SQLITE_OK ) abendError("lookaside configuration filed: %d", rc); + } + #ifndef SQLITE_OMIT_TRACE + sqlite3_trace(db, verboseFlag ? traceCallback : traceNoop, 0); + #endif + sqlite3_create_function(db, "eval", 1, SQLITE_UTF8, 0, sqlEvalFunc, 0, 0); + sqlite3_create_function(db, "eval", 2, SQLITE_UTF8, 0, sqlEvalFunc, 0, 0); + sqlite3_create_module(db, "generate_series", &seriesModule, 0); + sqlite3_limit(db, SQLITE_LIMIT_LENGTH, 1000000); + if( zEncoding ) sqlexec(db, "PRAGMA encoding=%s", zEncoding); + if( pageSize ) sqlexec(db, "PRAGMA pagesize=%d", pageSize); + if( doAutovac ) sqlexec(db, "PRAGMA auto_vacuum=FULL"); + iStart = timeOfDay(); + + /* If using an input database file and that database contains a table + ** named "autoexec" with a column "sql", then replace the input SQL + ** with the concatenated text of the autoexec table. In this way, + ** if the database file is the input being fuzzed, the SQL text is + ** fuzzed at the same time. */ + if( sqlite3_table_column_metadata(db,0,"autoexec","sql",0,0,0,0,0)==0 ){ + sqlite3_stmt *pStmt2; + rc = sqlite3_prepare_v2(db,"SELECT sql FROM autoexec",-1,&pStmt2,0); + if( rc==SQLITE_OK ){ + while( sqlite3_step(pStmt2)==SQLITE_ROW ){ + StrAppend(&sql, (const char*)sqlite3_column_text(pStmt2, 0)); + StrAppend(&sql, "\n"); + } + } + sqlite3_finalize(pStmt2); + zSql = StrStr(&sql); + } + + g.bOomEnable = 1; + if( verboseFlag ){ + zErrMsg = 0; + rc = sqlite3_exec(db, zSql, execCallback, 0, &zErrMsg); + if( zErrMsg ){ + sqlite3_snprintf(sizeof(zErrBuf),zErrBuf,"%z", zErrMsg); + zErrMsg = 0; + } + }else { + rc = sqlite3_exec(db, zSql, execNoop, 0, 0); + } + g.bOomEnable = 0; + iEnd = timeOfDay(); + StrFree(&sql); + rc = sqlite3_close(db); + if( rc ){ + abendError("sqlite3_close() failed with rc=%d", rc); + } + if( !zDataOut && sqlite3_memory_used()>0 ){ + abendError("memory in use after close: %lld bytes",sqlite3_memory_used()); + } + if( oomFlag ){ + /* Limit the number of iterations of the OOM loop to OOM_MAX. If the + ** first pass (single failure) exceeds 2/3rds of OOM_MAX this skip the + ** second pass (continuous failure after first) completely. */ + if( g.nOomFault==0 || oomCnt>OOM_MAX ){ + if( g.bOomOnce && oomCnt<=(OOM_MAX*2/3) ){ + oomCnt = g.iOomCntdown = 1; + g.bOomOnce = 0; + }else{ + oomCnt = 0; + } + }else{ + g.iOomCntdown = ++oomCnt; + g.nOomFault = 0; + } + if( oomCnt ){ + if( verboseFlag ){ + printf("%s.%d\n", g.bOomOnce ? "Once" : "Multi", oomCnt); + fflush(stdout); + } + nTest++; + } + } + }while( oomCnt>0 ); + + /* Store unique test cases in the in the dataDb database if the + ** --unique-cases flag is present + */ + if( zDataOut ){ + sqlite3_bind_blob(pStmt, 1, &zIn[i], iNext-i, SQLITE_STATIC); + sqlite3_bind_int64(pStmt, 2, iEnd - iStart); + rc = sqlite3_step(pStmt); + if( rc!=SQLITE_DONE ) abendError("%s", sqlite3_errmsg(dataDb)); + sqlite3_reset(pStmt); + } + + /* Free the SQL from the current test case + */ + if( zToFree ){ + sqlite3_free(zToFree); + zToFree = 0; + } + zIn[iNext] = cSaved; + + /* Show test-case results in --verbose mode + */ + if( verboseFlag ){ + printf("RESULT-CODE: %d\n", rc); + if( zErrMsg ){ + printf("ERROR-MSG: [%s]\n", zErrBuf); + } + fflush(stdout); + } + + /* Simulate an error if the TEST_FAILURE environment variable is "5". + ** This is used to verify that automated test script really do spot + ** errors that occur in this test program. + */ + if( zFailCode ){ + if( zFailCode[0]=='5' && zFailCode[1]==0 ){ + abendError("simulated failure"); + }else if( zFailCode[0]!=0 ){ + /* If TEST_FAILURE is something other than 5, just exit the test + ** early */ + printf("\nExit early due to TEST_FAILURE being set"); + break; + } + } + } + if( !verboseFlag && multiTest && !quietFlag && !oomFlag ) printf("\n"); + } + + /* Report total number of tests run + */ + if( nTest>1 && !quietFlag ){ + sqlite3_int64 iElapse = timeOfDay() - iBegin; + printf("%s: 0 errors out of %d tests in %d.%03d seconds\nSQLite %s %s\n", + g.zArgv0, nTest, (int)(iElapse/1000), (int)(iElapse%1000), + sqlite3_libversion(), sqlite3_sourceid()); + } + + /* Write the unique test cases if the --unique-cases flag was used + */ + if( zDataOut ){ + int n = 0; + FILE *out = fopen(zDataOut, "wb"); + if( out==0 ) abendError("cannot open %s for writing", zDataOut); + if( nHeader>0 ) fwrite(zIn, nHeader, 1, out); + sqlite3_finalize(pStmt); + rc = sqlite3_prepare_v2(dataDb, "SELECT sql, tm FROM testcase ORDER BY tm, sql", + -1, &pStmt, 0); + if( rc ) abendError("%s", sqlite3_errmsg(dataDb)); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + fprintf(out,"/****<%d:%dms>****/", ++n, sqlite3_column_int(pStmt,1)); + fwrite(sqlite3_column_blob(pStmt,0),sqlite3_column_bytes(pStmt,0),1,out); + } + fclose(out); + sqlite3_finalize(pStmt); + sqlite3_close(dataDb); + } + + /* Clean up and exit. + */ + free(azInFile); + free(zIn); + free(pHeap); + free(pLook); + free(pScratch); + free(pPCache); + return 0; +} diff --git a/tool/genfkey.README b/tool/genfkey.README new file mode 100644 index 0000000..57cdff8 --- /dev/null +++ b/tool/genfkey.README @@ -0,0 +1,137 @@ + +OVERVIEW + + The SQLite library is capable of parsing SQL foreign key constraints + supplied as part of CREATE TABLE statements, but it does not actually + implement them. However, most of the features of foreign keys may be + implemented using SQL triggers, which SQLite does support. This text + file describes a feature of the SQLite shell tool (sqlite3) that + extracts foreign key definitions from an existing SQLite database and + creates the set of CREATE TRIGGER statements required to implement + the foreign key constraints. + +CAPABILITIES + + An SQL foreign key is a constraint that requires that each row in + the "child" table corresponds to a row in the "parent" table. For + example, the following schema: + + CREATE TABLE parent(a, b, c, PRIMARY KEY(a, b)); + CREATE TABLE child(d, e, f, FOREIGN KEY(d, e) REFERENCES parent(a, b)); + + implies that for each row in table "child", there must be a row in + "parent" for which the expression (child.d==parent.a AND child.e==parent.b) + is true. The columns in the parent table are required to be either the + primary key columns or subject to a UNIQUE constraint. There is no such + requirement for the columns of the child table. + + At this time, all foreign keys are implemented as if they were + "MATCH NONE", even if the declaration specified "MATCH PARTIAL" or + "MATCH FULL". "MATCH NONE" means that if any of the key columns in + the child table are NULL, then there is no requirement for a corresponding + row in the parent table. So, taking this into account, the expression that + must be true for every row of the child table in the above example is + actually: + + (child.d IS NULL) OR + (child.e IS NULL) OR + (child.d==parent.a AND child.e==parent.b) + + Attempting to insert or update a row in the child table so that the + affected row violates this constraint results in an exception being + thrown. + + The effect of attempting to delete or update a row in the parent table + so that the constraint becomes untrue for one or more rows in the child + table depends on the "ON DELETE" or "ON UPDATE" actions specified as + part of the foreign key definition, respectively. Three different actions + are supported: "RESTRICT" (the default), "CASCADE" and "SET NULL". SQLite + will also parse the "SET DEFAULT" action, but this is not implemented + and "RESTRICT" is used instead. + + RESTRICT: Attempting to update or delete a row in the parent table so + that the constraint becomes untrue for one or more rows in + the child table is not allowed. An exception is thrown. + + CASCADE: Instead of throwing an exception, all corresponding child table + rows are either deleted (if the parent row is being deleted) + or updated to match the new parent key values (if the parent + row is being updated). + + SET NULL: Instead of throwing an exception, the foreign key fields of + all corresponding child table rows are set to NULL. + +LIMITATIONS + + Apart from those limitiations described above: + + * Implicit mapping to composite primary keys is not supported. If + a parent table has a composite primary key, then any child table + that refers to it must explicitly map each column. For example, given + the following definition of table "parent": + + CREATE TABLE parent(a, b, c, PRIMARY KEY(a, b)); + + only the first of the following two definitions of table "child" + is supported: + + CREATE TABLE child(d, e, f, FOREIGN KEY(d, e) REFERENCES parent(a, b)); + CREATE TABLE child(d, e, f, FOREIGN KEY(d, e) REFERENCES parent); + + An implicit reference to a composite primary key is detected as an + error when the program is run (see below). + + * SQLite does not support recursive triggers, and therefore this program + does not support recursive CASCADE or SET NULL foreign key + relationships. If the parent and the child tables of a CASCADE or + SET NULL foreign key are the same table, the generated triggers will + malfunction. This is also true if the recursive foreign key constraint + is indirect (for example if table A references table B which references + table A with a CASCADE or SET NULL foreign key constraint). + + Recursive CASCADE or SET NULL foreign key relationships are *not* + detected as errors when the program is run. Buyer beware. + +USAGE + + The functionality is accessed through an sqlite3 shell tool "dot-command": + + .genfkey ?--no-drop? ?--ignore-errors? ?--exec? + + When this command is run, it first checks the schema of the open SQLite + database for foreign key related errors or inconsistencies. For example, + a foreign key that refers to a parent table that does not exist, or + a foreign key that refers to columns in a parent table that are not + guaranteed to be unique. If such errors are found and the --ignore-errors + option was not present, a message for each one is printed to stderr and + no further processing takes place. + + If errors are found and the --ignore-errors option is passed, then + no error messages are printed. No "CREATE TRIGGER" statements are generated + for foriegn-key definitions that contained errors, they are silently + ignored by subsequent processing. + + All triggers generated by this command have names that match the pattern + "genfkey*". Unless the --no-drop option is specified, then the program + also generates a "DROP TRIGGER" statement for each trigger that exists + in the database with a name that matches this pattern. This allows the + program to be used to upgrade a database schema for which foreign key + triggers have already been installed (i.e. after new tables are created + or existing tables dropped). + + Finally, a series of SQL trigger definitions (CREATE TRIGGER statements) + that implement the foreign key constraints found in the database schema are + generated. + + If the --exec option was passed, then all generated SQL is immediately + executed on the database. Otherwise, the generated SQL strings are output + in the same way as the results of SELECT queries are. Normally, this means + they will be printed to stdout, but this can be configured using other + dot-commands (i.e. ".output"). + + The simplest way to activate the foriegn key definitions in a database + is simply to open it using the shell tool and enter the command + ".genfkey --exec": + + sqlite> .genfkey --exec + diff --git a/tool/genfkey.test b/tool/genfkey.test new file mode 100644 index 0000000..16c67bb --- /dev/null +++ b/tool/genfkey.test @@ -0,0 +1,353 @@ + +package require sqlite3 + +proc do_test {name cmd expected} { + puts -nonewline "$name ..." + set res [uplevel $cmd] + if {$res eq $expected} { + puts Ok + } else { + puts Error + puts " Got: $res" + puts " Expected: $expected" + exit + } +} + +proc execsql {sql} { + uplevel [list db eval $sql] +} + +proc catchsql {sql} { + set rc [catch {uplevel [list db eval $sql]} msg] + list $rc $msg +} + +file delete -force test.db test.db.journal +sqlite3 db test.db + +# The following tests - genfkey-1.* - test RESTRICT foreign keys. +# +do_test genfkey-1.1 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, UNIQUE(b, c)); + CREATE TABLE t2(e REFERENCES t1, f); + CREATE TABLE t3(g, h, i, FOREIGN KEY (h, i) REFERENCES t1(b, c)); + } +} {} +do_test genfkey-1.2 { + execsql [exec ./sqlite3 test.db .genfkey] +} {} +do_test genfkey-1.3 { + catchsql { INSERT INTO t2 VALUES(1, 2) } +} {1 {constraint failed}} +do_test genfkey-1.4 { + execsql { + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t2 VALUES(1, 2); + } +} {} +do_test genfkey-1.5 { + execsql { INSERT INTO t2 VALUES(NULL, 3) } +} {} +do_test genfkey-1.6 { + catchsql { UPDATE t2 SET e = 5 WHERE e IS NULL } +} {1 {constraint failed}} +do_test genfkey-1.7 { + execsql { UPDATE t2 SET e = 1 WHERE e IS NULL } +} {} +do_test genfkey-1.8 { + execsql { UPDATE t2 SET e = NULL WHERE f = 3 } +} {} +do_test genfkey-1.9 { + catchsql { UPDATE t1 SET a = 10 } +} {1 {constraint failed}} +do_test genfkey-1.9a { + catchsql { UPDATE t1 SET a = NULL } +} {1 {datatype mismatch}} +do_test genfkey-1.10 { + catchsql { DELETE FROM t1 } +} {1 {constraint failed}} +do_test genfkey-1.11 { + execsql { UPDATE t2 SET e = NULL } +} {} +do_test genfkey-1.12 { + execsql { + UPDATE t1 SET a = 10 ; + DELETE FROM t1; + DELETE FROM t2; + } +} {} + +do_test genfkey-1.13 { + execsql { + INSERT INTO t3 VALUES(1, NULL, NULL); + INSERT INTO t3 VALUES(1, 2, NULL); + INSERT INTO t3 VALUES(1, NULL, 3); + } +} {} +do_test genfkey-1.14 { + catchsql { INSERT INTO t3 VALUES(3, 1, 4) } +} {1 {constraint failed}} +do_test genfkey-1.15 { + execsql { + INSERT INTO t1 VALUES(1, 1, 4); + INSERT INTO t3 VALUES(3, 1, 4); + } +} {} +do_test genfkey-1.16 { + catchsql { DELETE FROM t1 } +} {1 {constraint failed}} +do_test genfkey-1.17 { + catchsql { UPDATE t1 SET b = 10} +} {1 {constraint failed}} +do_test genfkey-1.18 { + execsql { UPDATE t1 SET a = 10} +} {} +do_test genfkey-1.19 { + catchsql { UPDATE t3 SET h = 'hello' WHERE i = 3} +} {1 {constraint failed}} + +do_test genfkey-1.X { + execsql { + DROP TABLE t1; + DROP TABLE t2; + DROP TABLE t3; + } +} {} + +# The following tests - genfkey-2.* - test CASCADE foreign keys. +# +do_test genfkey-2.1 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, UNIQUE(b, c)); + CREATE TABLE t2(e REFERENCES t1 ON UPDATE CASCADE ON DELETE CASCADE, f); + CREATE TABLE t3(g, h, i, + FOREIGN KEY (h, i) + REFERENCES t1(b, c) ON UPDATE CASCADE ON DELETE CASCADE + ); + } +} {} +do_test genfkey-2.2 { + execsql [exec ./sqlite3 test.db .genfkey] +} {} +do_test genfkey-2.3 { + execsql { + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + INSERT INTO t2 VALUES(1, 'one'); + INSERT INTO t2 VALUES(4, 'four'); + } +} {} +do_test genfkey-2.4 { + execsql { + UPDATE t1 SET a = 2 WHERE a = 1; + SELECT * FROM t2; + } +} {2 one 4 four} +do_test genfkey-2.5 { + execsql { + DELETE FROM t1 WHERE a = 4; + SELECT * FROM t2; + } +} {2 one} +do_test genfkey-2.6 { + execsql { + INSERT INTO t3 VALUES('hello', 2, 3); + UPDATE t1 SET c = 2; + SELECT * FROM t3; + } +} {hello 2 2} +do_test genfkey-2.7 { + execsql { + DELETE FROM t1; + SELECT * FROM t3; + } +} {} +do_test genfkey-2.X { + execsql { + DROP TABLE t1; + DROP TABLE t2; + DROP TABLE t3; + } +} {} + + +# The following tests - genfkey-3.* - test SET NULL foreign keys. +# +do_test genfkey-3.1 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, UNIQUE(c, b)); + CREATE TABLE t2(e REFERENCES t1 ON UPDATE SET NULL ON DELETE SET NULL, f); + CREATE TABLE t3(g, h, i, + FOREIGN KEY (h, i) + REFERENCES t1(b, c) ON UPDATE SET NULL ON DELETE SET NULL + ); + } +} {} +do_test genfkey-3.2 { + execsql [exec ./sqlite3 test.db .genfkey] +} {} +do_test genfkey-3.3 { + execsql { + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + INSERT INTO t2 VALUES(1, 'one'); + INSERT INTO t2 VALUES(4, 'four'); + } +} {} +do_test genfkey-3.4 { + execsql { + UPDATE t1 SET a = 2 WHERE a = 1; + SELECT * FROM t2; + } +} {{} one 4 four} +do_test genfkey-3.5 { + execsql { + DELETE FROM t1 WHERE a = 4; + SELECT * FROM t2; + } +} {{} one {} four} +do_test genfkey-3.6 { + execsql { + INSERT INTO t3 VALUES('hello', 2, 3); + UPDATE t1 SET c = 2; + SELECT * FROM t3; + } +} {hello {} {}} +do_test genfkey-2.7 { + execsql { + UPDATE t3 SET h = 2, i = 2; + DELETE FROM t1; + SELECT * FROM t3; + } +} {hello {} {}} +do_test genfkey-3.X { + execsql { + DROP TABLE t1; + DROP TABLE t2; + DROP TABLE t3; + } +} {} + +# The following tests - genfkey-4.* - test that errors in the schema +# are detected correctly. +# +do_test genfkey-4.1 { + execsql { + CREATE TABLE t1(a REFERENCES nosuchtable, b); + CREATE TABLE t2(a REFERENCES t1, b); + + CREATE TABLE t3(a, b, c, PRIMARY KEY(a, b)); + CREATE TABLE t4(a, b, c, FOREIGN KEY(c, b) REFERENCES t3); + + CREATE TABLE t5(a REFERENCES t4(d), b, c); + CREATE TABLE t6(a REFERENCES t4(a), b, c); + CREATE TABLE t7(a REFERENCES t3(a), b, c); + CREATE TABLE t8(a REFERENCES nosuchtable(a), b, c); + } +} {} + +do_test genfkey-4.X { + set rc [catch {exec ./sqlite3 test.db .genfkey} msg] + list $rc $msg +} "1 {[string trim { +Error in table t5: foreign key columns do not exist +Error in table t8: foreign key columns do not exist +Error in table t4: implicit mapping to composite primary key +Error in table t1: implicit mapping to non-existant primary key +Error in table t2: implicit mapping to non-existant primary key +Error in table t6: foreign key is not unique +Error in table t7: foreign key is not unique +}]}" + +# Test that ticket #3800 has been resolved. +# +do_test genfkey-5.1 { + execsql { + DROP TABLE t1; DROP TABLE t2; DROP TABLE t3; + DROP TABLE t4; DROP TABLE t5; DROP TABLE t6; + DROP TABLE t7; DROP TABLE t8; + } +} {} +do_test genfkey-5.2 { + execsql { + CREATE TABLE "t.3" (c1 PRIMARY KEY); + CREATE TABLE t13 (c1, foreign key(c1) references "t.3"(c1)); + } +} {} +do_test genfkey-5.3 { + set rc [catch {exec ./sqlite3 test.db .genfkey} msg] +} {0} +do_test genfkey-5.4 { + db eval $msg +} {} +do_test genfkey-5.5 { + catchsql { INSERT INTO t13 VALUES(1) } +} {1 {constraint failed}} +do_test genfkey-5.5 { + catchsql { + INSERT INTO "t.3" VALUES(1); + INSERT INTO t13 VALUES(1); + } +} {0 {}} + +# Test also column names that require quoting. +do_test genfkey-6.1 { + execsql { + DROP TABLE "t.3"; + DROP TABLE t13; + CREATE TABLE p( + "a.1 first", "b.2 second", + UNIQUE("a.1 first", "b.2 second") + ); + CREATE TABLE c( + "c.1 I", "d.2 II", + FOREIGN KEY("c.1 I", "d.2 II") + REFERENCES p("a.1 first", "b.2 second") + ON UPDATE CASCADE ON DELETE CASCADE + ); + } +} {} +do_test genfkey-6.2 { + set rc [catch {exec ./sqlite3 test.db .genfkey} msg] +} {0} +do_test genfkey-6.3 { + execsql $msg + execsql { + INSERT INTO p VALUES('A', 'B'); + INSERT INTO p VALUES('C', 'D'); + INSERT INTO c VALUES('A', 'B'); + INSERT INTO c VALUES('C', 'D'); + UPDATE p SET "a.1 first" = 'X' WHERE rowid = 1; + DELETE FROM p WHERE rowid = 2; + } + execsql { SELECT * FROM c } +} {X B} + +do_test genfkey-6.4 { + execsql { + DROP TABLE p; + DROP TABLE c; + CREATE TABLE parent("a.1", PRIMARY KEY("a.1")); + CREATE TABLE child("b.2", FOREIGN KEY("b.2") REFERENCES parent("a.1")); + } + set rc [catch {exec ./sqlite3 test.db .genfkey} msg] +} {0} +do_test genfkey-6.5 { + execsql $msg + execsql { + INSERT INTO parent VALUES(1); + INSERT INTO child VALUES(1); + } + catchsql { UPDATE parent SET "a.1"=0 } +} {1 {constraint failed}} +do_test genfkey-6.6 { + catchsql { UPDATE child SET "b.2"=7 } +} {1 {constraint failed}} +do_test genfkey-6.7 { + execsql { + SELECT * FROM parent; + SELECT * FROM child; + } +} {1 1} diff --git a/tool/getlock.c b/tool/getlock.c new file mode 100644 index 0000000..7eff04d --- /dev/null +++ b/tool/getlock.c @@ -0,0 +1,134 @@ +/* +** This utility program looks at an SQLite database and determines whether +** or not it is locked, the kind of lock, and who is holding this lock. +** +** This only works on unix when the posix advisory locking method is used +** (which is the default on unix) and when the PENDING_BYTE is in its +** usual place. +*/ +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> +#include <fcntl.h> +#include <string.h> +#include <stdio.h> +#include <stdlib.h> +#include <errno.h> + +static void usage(const char *argv0){ + fprintf(stderr, "Usage: %s database\n", argv0); + exit(1); +} + +/* Check for a conflicting lock. If one is found, print an this +** on standard output using the format string given and return 1. +** If there are no conflicting locks, return 0. +*/ +static int isLocked( + int h, /* File descriptor to check */ + int type, /* F_RDLCK or F_WRLCK */ + unsigned int iOfst, /* First byte of the lock */ + unsigned int iCnt, /* Number of bytes in the lock range */ + const char *zType /* Type of lock */ +){ + struct flock lk; + + memset(&lk, 0, sizeof(lk)); + lk.l_type = type; + lk.l_whence = SEEK_SET; + lk.l_start = iOfst; + lk.l_len = iCnt; + if( fcntl(h, F_GETLK, &lk)==(-1) ){ + fprintf(stderr, "fcntl(%d) failed: errno=%d\n", h, errno); + exit(1); + } + if( lk.l_type==F_UNLCK ) return 0; + printf("%s lock held by %d\n", zType, (int)lk.l_pid); + return 1; +} + +/* +** Location of locking bytes in the database file +*/ +#define PENDING_BYTE (0x40000000) +#define RESERVED_BYTE (PENDING_BYTE+1) +#define SHARED_FIRST (PENDING_BYTE+2) +#define SHARED_SIZE 510 + +/* +** Lock locations for shared-memory locks used by WAL mode. +*/ +#define SHM_BASE 120 +#define SHM_WRITE SHM_BASE +#define SHM_CHECKPOINT (SHM_BASE+1) +#define SHM_RECOVER (SHM_BASE+2) +#define SHM_READ_FIRST (SHM_BASE+3) +#define SHM_READ_SIZE 5 + + +int main(int argc, char **argv){ + int hDb; /* File descriptor for the open database file */ + int hShm; /* File descriptor for WAL shared-memory file */ + char *zShm; /* Name of the shared-memory file for WAL mode */ + ssize_t got; /* Bytes read from header */ + int isWal; /* True if in WAL mode */ + int nName; /* Length of filename */ + unsigned char aHdr[100]; /* Database header */ + int nLock = 0; /* Number of locks held */ + int i; /* Loop counter */ + + if( argc!=2 ) usage(argv[0]); + hDb = open(argv[1], O_RDONLY, 0); + if( hDb<0 ){ + fprintf(stderr, "cannot open %s\n", argv[1]); + return 1; + } + + /* Make sure we are dealing with an database file */ + got = read(hDb, aHdr, 100); + if( got!=100 || memcmp(aHdr, "SQLite format 3",16)!=0 ){ + fprintf(stderr, "not an SQLite database: %s\n", argv[1]); + exit(1); + } + + /* First check for an exclusive lock */ + if( isLocked(hDb, F_RDLCK, SHARED_FIRST, SHARED_SIZE, "EXCLUSIVE") ){ + return 0; + } + isWal = aHdr[18]==2; + if( isWal==0 ){ + /* Rollback mode */ + if( isLocked(hDb, F_RDLCK, PENDING_BYTE, 1, "PENDING") ) return 0; + if( isLocked(hDb, F_RDLCK, RESERVED_BYTE, 1, "RESERVED") ) return 0; + if( isLocked(hDb, F_WRLCK, SHARED_FIRST, SHARED_SIZE, "SHARED") ){ + return 0; + } + }else{ + /* WAL mode */ + nName = (int)strlen(argv[1]); + zShm = malloc( nName + 100 ); + if( zShm==0 ){ + fprintf(stderr, "out of memory\n"); + exit(1); + } + memcpy(zShm, argv[1], nName); + memcpy(&zShm[nName], "-shm", 5); + hShm = open(zShm, O_RDONLY, 0); + if( hShm<0 ){ + fprintf(stderr, "cannot open %s\n", zShm); + return 1; + } + if( isLocked(hShm, F_RDLCK, SHM_RECOVER, 1, "WAL-RECOVERY") ){ + return 0; + } + nLock += isLocked(hShm, F_RDLCK, SHM_CHECKPOINT, 1, "WAL-CHECKPOINT"); + nLock += isLocked(hShm, F_RDLCK, SHM_WRITE, 1, "WAL-WRITE"); + for(i=0; i<SHM_READ_SIZE; i++){ + nLock += isLocked(hShm, F_WRLCK, SHM_READ_FIRST+i, 1, "WAL-READ"); + } + } + if( nLock==0 ){ + printf("file is not locked\n"); + } + return 0; +} diff --git a/tool/index_usage.c b/tool/index_usage.c new file mode 100644 index 0000000..9bd3c9f --- /dev/null +++ b/tool/index_usage.c @@ -0,0 +1,233 @@ +/* +** 2018-12-04 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements a utility program used to help determine which +** indexes in a database schema are used and unused, and how often specific +** indexes are used. +*/ +#include "sqlite3.h" +#include <stdio.h> +#include <stdlib.h> +#include <assert.h> +#include <string.h> + +static void usage(const char *argv0){ + printf("Usage: %s [OPTIONS] DATABASE LOG\n\n", argv0); + printf( + "DATABASE is an SQLite database against which various statements\n" + "have been run. The SQL text is stored in LOG. LOG is an SQLite\n" + "database with this schema:\n" + "\n" + " CREATE TABLE sqllog(sql TEXT);\n" + "\n" + "This utility program analyzes statements contained in LOG and prints\n" + "a report showing how many times each index in DATABASE is used by the\n" + "statements in LOG.\n" + "\n" + "DATABASE only needs to contain the schema used by the statements in\n" + "LOG. The content can be removed from DATABASE.\n" + ); + printf( + "\nOPTIONS:\n\n" + " --progress N Show a progress message after every N input rows\n" + " -q Omit error message when parsing log entries\n" + " --using NAME Print SQL statements that use index NAME\n" + ); + printf("\nAnalysis will be done by SQLite version %s dated %.20s\n" + "checkin number %.40s. Different versions\n" + "of SQLite might use different indexes.\n", + sqlite3_libversion(), sqlite3_sourceid(), sqlite3_sourceid()+21); + exit(1); +} + +int main(int argc, char **argv){ + sqlite3 *db = 0; /* The main database */ + sqlite3_stmt *pStmt = 0; /* a query */ + char *zSql; + int nErr = 0; + int rc; + int bQuiet = 0; + int i, j; + const char *zUsing = 0; + sqlite3_stmt *pIncrCnt = 0; + int nRow = 0; + int iProgress = 0; + + for(i=j=1; i<argc; i++){ + const char *z = argv[i]; + if( z[0]=='-' ){ + z++; + if( z[0]=='-' ) z++; + if( strcmp(z,"progress")==0 ){ + if( i+1<argc ){ + iProgress = strtol(argv[++i],0,0); + continue; + } + printf("The --progress option requires an argument\n"); + exit(0); + } + if( strcmp(z,"q")==0 ){ + bQuiet = 1; + continue; + } + if( strcmp(z,"using")==0 ){ + if( i+1<argc ){ + zUsing = argv[++i]; + continue; + } + printf("The --using option requires an argument\n"); + exit(0); + } + if( strcmp(z, "help")==0 || strcmp(z, "?")==0 ){ + usage(argv[0]); + } + printf("Unknown command-line option: \"%s\"\n", argv[i]); + exit(0); + }else{ + if( j<i ) argv[j++] = argv[i]; + } + } + argc = j; + + if( argc!=3 ) usage(argv[0]); + rc = sqlite3_open_v2(argv[1], &db, SQLITE_OPEN_READONLY, 0); + if( rc ){ + printf("Cannot open \"%s\" for reading: %s\n", argv[1], sqlite3_errmsg(db)); + goto errorOut; + } + rc = sqlite3_prepare_v2(db, "SELECT * FROM sqlite_schema", -1, &pStmt, 0); + if( rc ){ + printf("Cannot read the schema from \"%s\" - %s\n", argv[1], + sqlite3_errmsg(db)); + goto errorOut; + } + sqlite3_finalize(pStmt); + pStmt = 0; + rc = sqlite3_exec(db, + "CREATE TABLE temp.idxu(\n" + " tbl TEXT COLLATE nocase,\n" + " idx TEXT COLLATE nocase,\n" + " cnt INT,\n" + " PRIMARY KEY(idx)\n" + ") WITHOUT ROWID;", 0, 0, 0); + if( rc ){ + printf("Cannot create the result table - %s\n", + sqlite3_errmsg(db)); + goto errorOut; + } + rc = sqlite3_exec(db, + "INSERT INTO temp.idxu(tbl,idx,cnt)" + " SELECT tbl_name, name, 0 FROM sqlite_schema" + " WHERE type='index' AND sql IS NOT NULL", 0, 0, 0); + + /* Open the LOG database */ + zSql = sqlite3_mprintf("ATTACH %Q AS log", argv[2]); + rc = sqlite3_exec(db, zSql, 0, 0, 0); + sqlite3_free(zSql); + if( rc ){ + printf("Cannot open the LOG database \"%s\" - %s\n", + argv[2], sqlite3_errmsg(db)); + goto errorOut; + } + rc = sqlite3_prepare_v2(db, + "SELECT sql, rowid FROM log.sqllog" + " WHERE upper(substr(sql,1,5)) NOT IN ('BEGIN','COMMI','ROLLB','PRAGM')", + -1, &pStmt, 0); + if( rc ){ + printf("Cannot read the SQLLOG table in the LOG database \"%s\" - %s\n", + argv[2], sqlite3_errmsg(db)); + goto errorOut; + } + + rc = sqlite3_prepare_v2(db, + "UPDATE temp.idxu SET cnt=cnt+1 WHERE idx=?1", + -1, &pIncrCnt, 0); + if( rc ){ + printf("Cannot prepare a statement to increment a counter for " + "indexes used\n"); + goto errorOut; + } + + /* Update the counts based on LOG */ + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + const char *zLog = (const char*)sqlite3_column_text(pStmt, 0); + sqlite3_stmt *pS2; + if( zLog==0 ) continue; + zSql = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zLog); + rc = sqlite3_prepare_v2(db, zSql, -1, &pS2, 0); + sqlite3_free(zSql); + if( rc ){ + if( !bQuiet ){ + printf("Cannot compile LOG entry %d (%s): %s\n", + sqlite3_column_int(pStmt, 1), zLog, sqlite3_errmsg(db)); + fflush(stdout); + } + nErr++; + }else{ + nRow++; + if( iProgress>0 && (nRow%iProgress)==0 ){ + printf("%d...\n", nRow); + fflush(stdout); + } + while( sqlite3_step(pS2)==SQLITE_ROW ){ + const char *zExplain = (const char*)sqlite3_column_text(pS2,3); + const char *z1, *z2; + int n; + /* printf("EXPLAIN: %s\n", zExplain); */ + z1 = strstr(zExplain, " USING INDEX "); + if( z1==0 ) continue; + z1 += 13; + for(z2=z1+1; z2[0] && z2[1]!='('; z2++){} + n = z2 - z1; + if( zUsing && sqlite3_strnicmp(zUsing, z1, n)==0 ){ + printf("Using %s:\n%s\n", zUsing, zLog); + fflush(stdout); + } + sqlite3_bind_text(pIncrCnt,1,z1,n,SQLITE_STATIC); + sqlite3_step(pIncrCnt); + sqlite3_reset(pIncrCnt); + } + } + sqlite3_finalize(pS2); + } + sqlite3_finalize(pStmt); + + /* Generate the report */ + rc = sqlite3_prepare_v2(db, + "SELECT tbl, idx, cnt, " + " (SELECT group_concat(name,',') FROM pragma_index_info(idx))" + " FROM temp.idxu, main.sqlite_schema" + " WHERE temp.idxu.tbl=main.sqlite_schema.tbl_name" + " AND temp.idxu.idx=main.sqlite_schema.name" + " ORDER BY cnt DESC, tbl, idx", + -1, &pStmt, 0); + if( rc ){ + printf("Cannot query the result table - %s\n", + sqlite3_errmsg(db)); + goto errorOut; + } + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + printf("%10d %s on %s(%s)\n", + sqlite3_column_int(pStmt, 2), + sqlite3_column_text(pStmt, 1), + sqlite3_column_text(pStmt, 0), + sqlite3_column_text(pStmt, 3)); + } + sqlite3_finalize(pStmt); + pStmt = 0; + +errorOut: + sqlite3_finalize(pIncrCnt); + sqlite3_finalize(pStmt); + sqlite3_close(db); + return nErr; +} diff --git a/tool/kvtest-speed.sh b/tool/kvtest-speed.sh new file mode 100644 index 0000000..5f2c834 --- /dev/null +++ b/tool/kvtest-speed.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# +# A script for running speed tests using kvtest. +# +# The test database must be set up first. Recommended +# command-line: +# +# ./kvtest init kvtest.db --count 100K --size 12K --variance 5K + +if test "$1" = "" +then + echo "Usage: $0 OUTPUTFILE [OPTIONS]" + exit +fi +NAME=$1 +shift +OPTS="-DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DIRECT_OVERFLOW_READ -DUSE_PREAD" +KVARGS="--count 100K --stats" +gcc -g -Os -I. $OPTS $* kvtest.c sqlite3.c -o kvtest + +# First run using SQL +rm cachegrind.out.[1-9][0-9]* +valgrind --tool=cachegrind ./kvtest run kvtest.db $KVARGS 2>&1 | tee summary-kvtest-$NAME.txt +mv cachegrind.out.[1-9][0-9]* cachegrind.out.sql-$NAME +cg_anno.tcl cachegrind.out.sql-$NAME >cout-kvtest-sql-$NAME.txt + +# Second run using the sqlite3_blob object +valgrind --tool=cachegrind ./kvtest run kvtest.db $KVARGS --blob-api 2>&1 | tee -a summary-kvtest-$NAME.txt +mv cachegrind.out.[1-9][0-9]* cachegrind.out.$NAME +cg_anno.tcl cachegrind.out.$NAME >cout-kvtest-$NAME.txt + +# Diff the sqlite3_blob API analysis for non-trunk runs. +if test "$NAME" != "trunk"; then + fossil test-diff --tk cout-kvtest-trunk.txt cout-kvtest-$NAME.txt & +fi diff --git a/tool/lemon.c b/tool/lemon.c new file mode 100644 index 0000000..7804837 --- /dev/null +++ b/tool/lemon.c @@ -0,0 +1,5916 @@ +/* +** This file contains all sources (including headers) to the LEMON +** LALR(1) parser generator. The sources have been combined into a +** single file to make it easy to include LEMON in the source tree +** and Makefile of another program. +** +** The author of this program disclaims copyright. +*/ +#include <stdio.h> +#include <stdarg.h> +#include <string.h> +#include <ctype.h> +#include <stdlib.h> +#include <assert.h> + +#define ISSPACE(X) isspace((unsigned char)(X)) +#define ISDIGIT(X) isdigit((unsigned char)(X)) +#define ISALNUM(X) isalnum((unsigned char)(X)) +#define ISALPHA(X) isalpha((unsigned char)(X)) +#define ISUPPER(X) isupper((unsigned char)(X)) +#define ISLOWER(X) islower((unsigned char)(X)) + + +#ifndef __WIN32__ +# if defined(_WIN32) || defined(WIN32) +# define __WIN32__ +# endif +#endif + +#ifdef __WIN32__ +#ifdef __cplusplus +extern "C" { +#endif +extern int access(const char *path, int mode); +#ifdef __cplusplus +} +#endif +#else +#include <unistd.h> +#endif + +/* #define PRIVATE static */ +#define PRIVATE + +#ifdef TEST +#define MAXRHS 5 /* Set low to exercise exception code */ +#else +#define MAXRHS 1000 +#endif + +extern void memory_error(); +static int showPrecedenceConflict = 0; +static char *msort(char*,char**,int(*)(const char*,const char*)); + +/* +** Compilers are getting increasingly pedantic about type conversions +** as C evolves ever closer to Ada.... To work around the latest problems +** we have to define the following variant of strlen(). +*/ +#define lemonStrlen(X) ((int)strlen(X)) + +/* +** Compilers are starting to complain about the use of sprintf() and strcpy(), +** saying they are unsafe. So we define our own versions of those routines too. +** +** There are three routines here: lemon_sprintf(), lemon_vsprintf(), and +** lemon_addtext(). The first two are replacements for sprintf() and vsprintf(). +** The third is a helper routine for vsnprintf() that adds texts to the end of a +** buffer, making sure the buffer is always zero-terminated. +** +** The string formatter is a minimal subset of stdlib sprintf() supporting only +** a few simply conversions: +** +** %d +** %s +** %.*s +** +*/ +static void lemon_addtext( + char *zBuf, /* The buffer to which text is added */ + int *pnUsed, /* Slots of the buffer used so far */ + const char *zIn, /* Text to add */ + int nIn, /* Bytes of text to add. -1 to use strlen() */ + int iWidth /* Field width. Negative to left justify */ +){ + if( nIn<0 ) for(nIn=0; zIn[nIn]; nIn++){} + while( iWidth>nIn ){ zBuf[(*pnUsed)++] = ' '; iWidth--; } + if( nIn==0 ) return; + memcpy(&zBuf[*pnUsed], zIn, nIn); + *pnUsed += nIn; + while( (-iWidth)>nIn ){ zBuf[(*pnUsed)++] = ' '; iWidth++; } + zBuf[*pnUsed] = 0; +} +static int lemon_vsprintf(char *str, const char *zFormat, va_list ap){ + int i, j, k, c; + int nUsed = 0; + const char *z; + char zTemp[50]; + str[0] = 0; + for(i=j=0; (c = zFormat[i])!=0; i++){ + if( c=='%' ){ + int iWidth = 0; + lemon_addtext(str, &nUsed, &zFormat[j], i-j, 0); + c = zFormat[++i]; + if( ISDIGIT(c) || (c=='-' && ISDIGIT(zFormat[i+1])) ){ + if( c=='-' ) i++; + while( ISDIGIT(zFormat[i]) ) iWidth = iWidth*10 + zFormat[i++] - '0'; + if( c=='-' ) iWidth = -iWidth; + c = zFormat[i]; + } + if( c=='d' ){ + int v = va_arg(ap, int); + if( v<0 ){ + lemon_addtext(str, &nUsed, "-", 1, iWidth); + v = -v; + }else if( v==0 ){ + lemon_addtext(str, &nUsed, "0", 1, iWidth); + } + k = 0; + while( v>0 ){ + k++; + zTemp[sizeof(zTemp)-k] = (v%10) + '0'; + v /= 10; + } + lemon_addtext(str, &nUsed, &zTemp[sizeof(zTemp)-k], k, iWidth); + }else if( c=='s' ){ + z = va_arg(ap, const char*); + lemon_addtext(str, &nUsed, z, -1, iWidth); + }else if( c=='.' && memcmp(&zFormat[i], ".*s", 3)==0 ){ + i += 2; + k = va_arg(ap, int); + z = va_arg(ap, const char*); + lemon_addtext(str, &nUsed, z, k, iWidth); + }else if( c=='%' ){ + lemon_addtext(str, &nUsed, "%", 1, 0); + }else{ + fprintf(stderr, "illegal format\n"); + exit(1); + } + j = i+1; + } + } + lemon_addtext(str, &nUsed, &zFormat[j], i-j, 0); + return nUsed; +} +static int lemon_sprintf(char *str, const char *format, ...){ + va_list ap; + int rc; + va_start(ap, format); + rc = lemon_vsprintf(str, format, ap); + va_end(ap); + return rc; +} +static void lemon_strcpy(char *dest, const char *src){ + while( (*(dest++) = *(src++))!=0 ){} +} +static void lemon_strcat(char *dest, const char *src){ + while( *dest ) dest++; + lemon_strcpy(dest, src); +} + + +/* a few forward declarations... */ +struct rule; +struct lemon; +struct action; + +static struct action *Action_new(void); +static struct action *Action_sort(struct action *); + +/********** From the file "build.h" ************************************/ +void FindRulePrecedences(struct lemon*); +void FindFirstSets(struct lemon*); +void FindStates(struct lemon*); +void FindLinks(struct lemon*); +void FindFollowSets(struct lemon*); +void FindActions(struct lemon*); + +/********* From the file "configlist.h" *********************************/ +void Configlist_init(void); +struct config *Configlist_add(struct rule *, int); +struct config *Configlist_addbasis(struct rule *, int); +void Configlist_closure(struct lemon *); +void Configlist_sort(void); +void Configlist_sortbasis(void); +struct config *Configlist_return(void); +struct config *Configlist_basis(void); +void Configlist_eat(struct config *); +void Configlist_reset(void); + +/********* From the file "error.h" ***************************************/ +void ErrorMsg(const char *, int,const char *, ...); + +/****** From the file "option.h" ******************************************/ +enum option_type { OPT_FLAG=1, OPT_INT, OPT_DBL, OPT_STR, + OPT_FFLAG, OPT_FINT, OPT_FDBL, OPT_FSTR}; +struct s_options { + enum option_type type; + const char *label; + char *arg; + const char *message; +}; +int OptInit(char**,struct s_options*,FILE*); +int OptNArgs(void); +char *OptArg(int); +void OptErr(int); +void OptPrint(void); + +/******** From the file "parse.h" *****************************************/ +void Parse(struct lemon *lemp); + +/********* From the file "plink.h" ***************************************/ +struct plink *Plink_new(void); +void Plink_add(struct plink **, struct config *); +void Plink_copy(struct plink **, struct plink *); +void Plink_delete(struct plink *); + +/********** From the file "report.h" *************************************/ +void Reprint(struct lemon *); +void ReportOutput(struct lemon *); +void ReportTable(struct lemon *, int, int); +void ReportHeader(struct lemon *); +void CompressTables(struct lemon *); +void ResortStates(struct lemon *); + +/********** From the file "set.h" ****************************************/ +void SetSize(int); /* All sets will be of size N */ +char *SetNew(void); /* A new set for element 0..N */ +void SetFree(char*); /* Deallocate a set */ +int SetAdd(char*,int); /* Add element to a set */ +int SetUnion(char *,char *); /* A <- A U B, thru element N */ +#define SetFind(X,Y) (X[Y]) /* True if Y is in set X */ + +/********** From the file "struct.h" *************************************/ +/* +** Principal data structures for the LEMON parser generator. +*/ + +typedef enum {LEMON_FALSE=0, LEMON_TRUE} Boolean; + +/* Symbols (terminals and nonterminals) of the grammar are stored +** in the following: */ +enum symbol_type { + TERMINAL, + NONTERMINAL, + MULTITERMINAL +}; +enum e_assoc { + LEFT, + RIGHT, + NONE, + UNK +}; +struct symbol { + const char *name; /* Name of the symbol */ + int index; /* Index number for this symbol */ + enum symbol_type type; /* Symbols are all either TERMINALS or NTs */ + struct rule *rule; /* Linked list of rules of this (if an NT) */ + struct symbol *fallback; /* fallback token in case this token doesn't parse */ + int prec; /* Precedence if defined (-1 otherwise) */ + enum e_assoc assoc; /* Associativity if precedence is defined */ + char *firstset; /* First-set for all rules of this symbol */ + Boolean lambda; /* True if NT and can generate an empty string */ + int useCnt; /* Number of times used */ + char *destructor; /* Code which executes whenever this symbol is + ** popped from the stack during error processing */ + int destLineno; /* Line number for start of destructor. Set to + ** -1 for duplicate destructors. */ + char *datatype; /* The data type of information held by this + ** object. Only used if type==NONTERMINAL */ + int dtnum; /* The data type number. In the parser, the value + ** stack is a union. The .yy%d element of this + ** union is the correct data type for this object */ + int bContent; /* True if this symbol ever carries content - if + ** it is ever more than just syntax */ + /* The following fields are used by MULTITERMINALs only */ + int nsubsym; /* Number of constituent symbols in the MULTI */ + struct symbol **subsym; /* Array of constituent symbols */ +}; + +/* Each production rule in the grammar is stored in the following +** structure. */ +struct rule { + struct symbol *lhs; /* Left-hand side of the rule */ + const char *lhsalias; /* Alias for the LHS (NULL if none) */ + int lhsStart; /* True if left-hand side is the start symbol */ + int ruleline; /* Line number for the rule */ + int nrhs; /* Number of RHS symbols */ + struct symbol **rhs; /* The RHS symbols */ + const char **rhsalias; /* An alias for each RHS symbol (NULL if none) */ + int line; /* Line number at which code begins */ + const char *code; /* The code executed when this rule is reduced */ + const char *codePrefix; /* Setup code before code[] above */ + const char *codeSuffix; /* Breakdown code after code[] above */ + struct symbol *precsym; /* Precedence symbol for this rule */ + int index; /* An index number for this rule */ + int iRule; /* Rule number as used in the generated tables */ + Boolean noCode; /* True if this rule has no associated C code */ + Boolean codeEmitted; /* True if the code has been emitted already */ + Boolean canReduce; /* True if this rule is ever reduced */ + Boolean doesReduce; /* Reduce actions occur after optimization */ + Boolean neverReduce; /* Reduce is theoretically possible, but prevented + ** by actions or other outside implementation */ + struct rule *nextlhs; /* Next rule with the same LHS */ + struct rule *next; /* Next rule in the global list */ +}; + +/* A configuration is a production rule of the grammar together with +** a mark (dot) showing how much of that rule has been processed so far. +** Configurations also contain a follow-set which is a list of terminal +** symbols which are allowed to immediately follow the end of the rule. +** Every configuration is recorded as an instance of the following: */ +enum cfgstatus { + COMPLETE, + INCOMPLETE +}; +struct config { + struct rule *rp; /* The rule upon which the configuration is based */ + int dot; /* The parse point */ + char *fws; /* Follow-set for this configuration only */ + struct plink *fplp; /* Follow-set forward propagation links */ + struct plink *bplp; /* Follow-set backwards propagation links */ + struct state *stp; /* Pointer to state which contains this */ + enum cfgstatus status; /* used during followset and shift computations */ + struct config *next; /* Next configuration in the state */ + struct config *bp; /* The next basis configuration */ +}; + +enum e_action { + SHIFT, + ACCEPT, + REDUCE, + ERROR, + SSCONFLICT, /* A shift/shift conflict */ + SRCONFLICT, /* Was a reduce, but part of a conflict */ + RRCONFLICT, /* Was a reduce, but part of a conflict */ + SH_RESOLVED, /* Was a shift. Precedence resolved conflict */ + RD_RESOLVED, /* Was reduce. Precedence resolved conflict */ + NOT_USED, /* Deleted by compression */ + SHIFTREDUCE /* Shift first, then reduce */ +}; + +/* Every shift or reduce operation is stored as one of the following */ +struct action { + struct symbol *sp; /* The look-ahead symbol */ + enum e_action type; + union { + struct state *stp; /* The new state, if a shift */ + struct rule *rp; /* The rule, if a reduce */ + } x; + struct symbol *spOpt; /* SHIFTREDUCE optimization to this symbol */ + struct action *next; /* Next action for this state */ + struct action *collide; /* Next action with the same hash */ +}; + +/* Each state of the generated parser's finite state machine +** is encoded as an instance of the following structure. */ +struct state { + struct config *bp; /* The basis configurations for this state */ + struct config *cfp; /* All configurations in this set */ + int statenum; /* Sequential number for this state */ + struct action *ap; /* List of actions for this state */ + int nTknAct, nNtAct; /* Number of actions on terminals and nonterminals */ + int iTknOfst, iNtOfst; /* yy_action[] offset for terminals and nonterms */ + int iDfltReduce; /* Default action is to REDUCE by this rule */ + struct rule *pDfltReduce;/* The default REDUCE rule. */ + int autoReduce; /* True if this is an auto-reduce state */ +}; +#define NO_OFFSET (-2147483647) + +/* A followset propagation link indicates that the contents of one +** configuration followset should be propagated to another whenever +** the first changes. */ +struct plink { + struct config *cfp; /* The configuration to which linked */ + struct plink *next; /* The next propagate link */ +}; + +/* The state vector for the entire parser generator is recorded as +** follows. (LEMON uses no global variables and makes little use of +** static variables. Fields in the following structure can be thought +** of as begin global variables in the program.) */ +struct lemon { + struct state **sorted; /* Table of states sorted by state number */ + struct rule *rule; /* List of all rules */ + struct rule *startRule; /* First rule */ + int nstate; /* Number of states */ + int nxstate; /* nstate with tail degenerate states removed */ + int nrule; /* Number of rules */ + int nruleWithAction; /* Number of rules with actions */ + int nsymbol; /* Number of terminal and nonterminal symbols */ + int nterminal; /* Number of terminal symbols */ + int minShiftReduce; /* Minimum shift-reduce action value */ + int errAction; /* Error action value */ + int accAction; /* Accept action value */ + int noAction; /* No-op action value */ + int minReduce; /* Minimum reduce action */ + int maxAction; /* Maximum action value of any kind */ + struct symbol **symbols; /* Sorted array of pointers to symbols */ + int errorcnt; /* Number of errors */ + struct symbol *errsym; /* The error symbol */ + struct symbol *wildcard; /* Token that matches anything */ + char *name; /* Name of the generated parser */ + char *arg; /* Declaration of the 3rd argument to parser */ + char *ctx; /* Declaration of 2nd argument to constructor */ + char *tokentype; /* Type of terminal symbols in the parser stack */ + char *vartype; /* The default type of non-terminal symbols */ + char *start; /* Name of the start symbol for the grammar */ + char *stacksize; /* Size of the parser stack */ + char *include; /* Code to put at the start of the C file */ + char *error; /* Code to execute when an error is seen */ + char *overflow; /* Code to execute on a stack overflow */ + char *failure; /* Code to execute on parser failure */ + char *accept; /* Code to execute when the parser excepts */ + char *extracode; /* Code appended to the generated file */ + char *tokendest; /* Code to execute to destroy token data */ + char *vardest; /* Code for the default non-terminal destructor */ + char *filename; /* Name of the input file */ + char *outname; /* Name of the current output file */ + char *tokenprefix; /* A prefix added to token names in the .h file */ + int nconflict; /* Number of parsing conflicts */ + int nactiontab; /* Number of entries in the yy_action[] table */ + int nlookaheadtab; /* Number of entries in yy_lookahead[] */ + int tablesize; /* Total table size of all tables in bytes */ + int basisflag; /* Print only basis configurations */ + int printPreprocessed; /* Show preprocessor output on stdout */ + int has_fallback; /* True if any %fallback is seen in the grammar */ + int nolinenosflag; /* True if #line statements should not be printed */ + int argc; /* Number of command-line arguments */ + char **argv; /* Command-line arguments */ +}; + +#define MemoryCheck(X) if((X)==0){ \ + extern void memory_error(); \ + memory_error(); \ +} + +/**************** From the file "table.h" *********************************/ +/* +** All code in this file has been automatically generated +** from a specification in the file +** "table.q" +** by the associative array code building program "aagen". +** Do not edit this file! Instead, edit the specification +** file, then rerun aagen. +*/ +/* +** Code for processing tables in the LEMON parser generator. +*/ +/* Routines for handling a strings */ + +const char *Strsafe(const char *); + +void Strsafe_init(void); +int Strsafe_insert(const char *); +const char *Strsafe_find(const char *); + +/* Routines for handling symbols of the grammar */ + +struct symbol *Symbol_new(const char *); +int Symbolcmpp(const void *, const void *); +void Symbol_init(void); +int Symbol_insert(struct symbol *, const char *); +struct symbol *Symbol_find(const char *); +struct symbol *Symbol_Nth(int); +int Symbol_count(void); +struct symbol **Symbol_arrayof(void); + +/* Routines to manage the state table */ + +int Configcmp(const char *, const char *); +struct state *State_new(void); +void State_init(void); +int State_insert(struct state *, struct config *); +struct state *State_find(struct config *); +struct state **State_arrayof(void); + +/* Routines used for efficiency in Configlist_add */ + +void Configtable_init(void); +int Configtable_insert(struct config *); +struct config *Configtable_find(struct config *); +void Configtable_clear(int(*)(struct config *)); + +/****************** From the file "action.c" *******************************/ +/* +** Routines processing parser actions in the LEMON parser generator. +*/ + +/* Allocate a new parser action */ +static struct action *Action_new(void){ + static struct action *actionfreelist = 0; + struct action *newaction; + + if( actionfreelist==0 ){ + int i; + int amt = 100; + actionfreelist = (struct action *)calloc(amt, sizeof(struct action)); + if( actionfreelist==0 ){ + fprintf(stderr,"Unable to allocate memory for a new parser action."); + exit(1); + } + for(i=0; i<amt-1; i++) actionfreelist[i].next = &actionfreelist[i+1]; + actionfreelist[amt-1].next = 0; + } + newaction = actionfreelist; + actionfreelist = actionfreelist->next; + return newaction; +} + +/* Compare two actions for sorting purposes. Return negative, zero, or +** positive if the first action is less than, equal to, or greater than +** the first +*/ +static int actioncmp( + struct action *ap1, + struct action *ap2 +){ + int rc; + rc = ap1->sp->index - ap2->sp->index; + if( rc==0 ){ + rc = (int)ap1->type - (int)ap2->type; + } + if( rc==0 && (ap1->type==REDUCE || ap1->type==SHIFTREDUCE) ){ + rc = ap1->x.rp->index - ap2->x.rp->index; + } + if( rc==0 ){ + rc = (int) (ap2 - ap1); + } + return rc; +} + +/* Sort parser actions */ +static struct action *Action_sort( + struct action *ap +){ + ap = (struct action *)msort((char *)ap,(char **)&ap->next, + (int(*)(const char*,const char*))actioncmp); + return ap; +} + +void Action_add( + struct action **app, + enum e_action type, + struct symbol *sp, + char *arg +){ + struct action *newaction; + newaction = Action_new(); + newaction->next = *app; + *app = newaction; + newaction->type = type; + newaction->sp = sp; + newaction->spOpt = 0; + if( type==SHIFT ){ + newaction->x.stp = (struct state *)arg; + }else{ + newaction->x.rp = (struct rule *)arg; + } +} +/********************** New code to implement the "acttab" module ***********/ +/* +** This module implements routines use to construct the yy_action[] table. +*/ + +/* +** The state of the yy_action table under construction is an instance of +** the following structure. +** +** The yy_action table maps the pair (state_number, lookahead) into an +** action_number. The table is an array of integers pairs. The state_number +** determines an initial offset into the yy_action array. The lookahead +** value is then added to this initial offset to get an index X into the +** yy_action array. If the aAction[X].lookahead equals the value of the +** of the lookahead input, then the value of the action_number output is +** aAction[X].action. If the lookaheads do not match then the +** default action for the state_number is returned. +** +** All actions associated with a single state_number are first entered +** into aLookahead[] using multiple calls to acttab_action(). Then the +** actions for that single state_number are placed into the aAction[] +** array with a single call to acttab_insert(). The acttab_insert() call +** also resets the aLookahead[] array in preparation for the next +** state number. +*/ +struct lookahead_action { + int lookahead; /* Value of the lookahead token */ + int action; /* Action to take on the given lookahead */ +}; +typedef struct acttab acttab; +struct acttab { + int nAction; /* Number of used slots in aAction[] */ + int nActionAlloc; /* Slots allocated for aAction[] */ + struct lookahead_action + *aAction, /* The yy_action[] table under construction */ + *aLookahead; /* A single new transaction set */ + int mnLookahead; /* Minimum aLookahead[].lookahead */ + int mnAction; /* Action associated with mnLookahead */ + int mxLookahead; /* Maximum aLookahead[].lookahead */ + int nLookahead; /* Used slots in aLookahead[] */ + int nLookaheadAlloc; /* Slots allocated in aLookahead[] */ + int nterminal; /* Number of terminal symbols */ + int nsymbol; /* total number of symbols */ +}; + +/* Return the number of entries in the yy_action table */ +#define acttab_lookahead_size(X) ((X)->nAction) + +/* The value for the N-th entry in yy_action */ +#define acttab_yyaction(X,N) ((X)->aAction[N].action) + +/* The value for the N-th entry in yy_lookahead */ +#define acttab_yylookahead(X,N) ((X)->aAction[N].lookahead) + +/* Free all memory associated with the given acttab */ +void acttab_free(acttab *p){ + free( p->aAction ); + free( p->aLookahead ); + free( p ); +} + +/* Allocate a new acttab structure */ +acttab *acttab_alloc(int nsymbol, int nterminal){ + acttab *p = (acttab *) calloc( 1, sizeof(*p) ); + if( p==0 ){ + fprintf(stderr,"Unable to allocate memory for a new acttab."); + exit(1); + } + memset(p, 0, sizeof(*p)); + p->nsymbol = nsymbol; + p->nterminal = nterminal; + return p; +} + +/* Add a new action to the current transaction set. +** +** This routine is called once for each lookahead for a particular +** state. +*/ +void acttab_action(acttab *p, int lookahead, int action){ + if( p->nLookahead>=p->nLookaheadAlloc ){ + p->nLookaheadAlloc += 25; + p->aLookahead = (struct lookahead_action *) realloc( p->aLookahead, + sizeof(p->aLookahead[0])*p->nLookaheadAlloc ); + if( p->aLookahead==0 ){ + fprintf(stderr,"malloc failed\n"); + exit(1); + } + } + if( p->nLookahead==0 ){ + p->mxLookahead = lookahead; + p->mnLookahead = lookahead; + p->mnAction = action; + }else{ + if( p->mxLookahead<lookahead ) p->mxLookahead = lookahead; + if( p->mnLookahead>lookahead ){ + p->mnLookahead = lookahead; + p->mnAction = action; + } + } + p->aLookahead[p->nLookahead].lookahead = lookahead; + p->aLookahead[p->nLookahead].action = action; + p->nLookahead++; +} + +/* +** Add the transaction set built up with prior calls to acttab_action() +** into the current action table. Then reset the transaction set back +** to an empty set in preparation for a new round of acttab_action() calls. +** +** Return the offset into the action table of the new transaction. +** +** If the makeItSafe parameter is true, then the offset is chosen so that +** it is impossible to overread the yy_lookaside[] table regardless of +** the lookaside token. This is done for the terminal symbols, as they +** come from external inputs and can contain syntax errors. When makeItSafe +** is false, there is more flexibility in selecting offsets, resulting in +** a smaller table. For non-terminal symbols, which are never syntax errors, +** makeItSafe can be false. +*/ +int acttab_insert(acttab *p, int makeItSafe){ + int i, j, k, n, end; + assert( p->nLookahead>0 ); + + /* Make sure we have enough space to hold the expanded action table + ** in the worst case. The worst case occurs if the transaction set + ** must be appended to the current action table + */ + n = p->nsymbol + 1; + if( p->nAction + n >= p->nActionAlloc ){ + int oldAlloc = p->nActionAlloc; + p->nActionAlloc = p->nAction + n + p->nActionAlloc + 20; + p->aAction = (struct lookahead_action *) realloc( p->aAction, + sizeof(p->aAction[0])*p->nActionAlloc); + if( p->aAction==0 ){ + fprintf(stderr,"malloc failed\n"); + exit(1); + } + for(i=oldAlloc; i<p->nActionAlloc; i++){ + p->aAction[i].lookahead = -1; + p->aAction[i].action = -1; + } + } + + /* Scan the existing action table looking for an offset that is a + ** duplicate of the current transaction set. Fall out of the loop + ** if and when the duplicate is found. + ** + ** i is the index in p->aAction[] where p->mnLookahead is inserted. + */ + end = makeItSafe ? p->mnLookahead : 0; + for(i=p->nAction-1; i>=end; i--){ + if( p->aAction[i].lookahead==p->mnLookahead ){ + /* All lookaheads and actions in the aLookahead[] transaction + ** must match against the candidate aAction[i] entry. */ + if( p->aAction[i].action!=p->mnAction ) continue; + for(j=0; j<p->nLookahead; j++){ + k = p->aLookahead[j].lookahead - p->mnLookahead + i; + if( k<0 || k>=p->nAction ) break; + if( p->aLookahead[j].lookahead!=p->aAction[k].lookahead ) break; + if( p->aLookahead[j].action!=p->aAction[k].action ) break; + } + if( j<p->nLookahead ) continue; + + /* No possible lookahead value that is not in the aLookahead[] + ** transaction is allowed to match aAction[i] */ + n = 0; + for(j=0; j<p->nAction; j++){ + if( p->aAction[j].lookahead<0 ) continue; + if( p->aAction[j].lookahead==j+p->mnLookahead-i ) n++; + } + if( n==p->nLookahead ){ + break; /* An exact match is found at offset i */ + } + } + } + + /* If no existing offsets exactly match the current transaction, find an + ** an empty offset in the aAction[] table in which we can add the + ** aLookahead[] transaction. + */ + if( i<end ){ + /* Look for holes in the aAction[] table that fit the current + ** aLookahead[] transaction. Leave i set to the offset of the hole. + ** If no holes are found, i is left at p->nAction, which means the + ** transaction will be appended. */ + i = makeItSafe ? p->mnLookahead : 0; + for(; i<p->nActionAlloc - p->mxLookahead; i++){ + if( p->aAction[i].lookahead<0 ){ + for(j=0; j<p->nLookahead; j++){ + k = p->aLookahead[j].lookahead - p->mnLookahead + i; + if( k<0 ) break; + if( p->aAction[k].lookahead>=0 ) break; + } + if( j<p->nLookahead ) continue; + for(j=0; j<p->nAction; j++){ + if( p->aAction[j].lookahead==j+p->mnLookahead-i ) break; + } + if( j==p->nAction ){ + break; /* Fits in empty slots */ + } + } + } + } + /* Insert transaction set at index i. */ +#if 0 + printf("Acttab:"); + for(j=0; j<p->nLookahead; j++){ + printf(" %d", p->aLookahead[j].lookahead); + } + printf(" inserted at %d\n", i); +#endif + for(j=0; j<p->nLookahead; j++){ + k = p->aLookahead[j].lookahead - p->mnLookahead + i; + p->aAction[k] = p->aLookahead[j]; + if( k>=p->nAction ) p->nAction = k+1; + } + if( makeItSafe && i+p->nterminal>=p->nAction ) p->nAction = i+p->nterminal+1; + p->nLookahead = 0; + + /* Return the offset that is added to the lookahead in order to get the + ** index into yy_action of the action */ + return i - p->mnLookahead; +} + +/* +** Return the size of the action table without the trailing syntax error +** entries. +*/ +int acttab_action_size(acttab *p){ + int n = p->nAction; + while( n>0 && p->aAction[n-1].lookahead<0 ){ n--; } + return n; +} + +/********************** From the file "build.c" *****************************/ +/* +** Routines to construction the finite state machine for the LEMON +** parser generator. +*/ + +/* Find a precedence symbol of every rule in the grammar. +** +** Those rules which have a precedence symbol coded in the input +** grammar using the "[symbol]" construct will already have the +** rp->precsym field filled. Other rules take as their precedence +** symbol the first RHS symbol with a defined precedence. If there +** are not RHS symbols with a defined precedence, the precedence +** symbol field is left blank. +*/ +void FindRulePrecedences(struct lemon *xp) +{ + struct rule *rp; + for(rp=xp->rule; rp; rp=rp->next){ + if( rp->precsym==0 ){ + int i, j; + for(i=0; i<rp->nrhs && rp->precsym==0; i++){ + struct symbol *sp = rp->rhs[i]; + if( sp->type==MULTITERMINAL ){ + for(j=0; j<sp->nsubsym; j++){ + if( sp->subsym[j]->prec>=0 ){ + rp->precsym = sp->subsym[j]; + break; + } + } + }else if( sp->prec>=0 ){ + rp->precsym = rp->rhs[i]; + } + } + } + } + return; +} + +/* Find all nonterminals which will generate the empty string. +** Then go back and compute the first sets of every nonterminal. +** The first set is the set of all terminal symbols which can begin +** a string generated by that nonterminal. +*/ +void FindFirstSets(struct lemon *lemp) +{ + int i, j; + struct rule *rp; + int progress; + + for(i=0; i<lemp->nsymbol; i++){ + lemp->symbols[i]->lambda = LEMON_FALSE; + } + for(i=lemp->nterminal; i<lemp->nsymbol; i++){ + lemp->symbols[i]->firstset = SetNew(); + } + + /* First compute all lambdas */ + do{ + progress = 0; + for(rp=lemp->rule; rp; rp=rp->next){ + if( rp->lhs->lambda ) continue; + for(i=0; i<rp->nrhs; i++){ + struct symbol *sp = rp->rhs[i]; + assert( sp->type==NONTERMINAL || sp->lambda==LEMON_FALSE ); + if( sp->lambda==LEMON_FALSE ) break; + } + if( i==rp->nrhs ){ + rp->lhs->lambda = LEMON_TRUE; + progress = 1; + } + } + }while( progress ); + + /* Now compute all first sets */ + do{ + struct symbol *s1, *s2; + progress = 0; + for(rp=lemp->rule; rp; rp=rp->next){ + s1 = rp->lhs; + for(i=0; i<rp->nrhs; i++){ + s2 = rp->rhs[i]; + if( s2->type==TERMINAL ){ + progress += SetAdd(s1->firstset,s2->index); + break; + }else if( s2->type==MULTITERMINAL ){ + for(j=0; j<s2->nsubsym; j++){ + progress += SetAdd(s1->firstset,s2->subsym[j]->index); + } + break; + }else if( s1==s2 ){ + if( s1->lambda==LEMON_FALSE ) break; + }else{ + progress += SetUnion(s1->firstset,s2->firstset); + if( s2->lambda==LEMON_FALSE ) break; + } + } + } + }while( progress ); + return; +} + +/* Compute all LR(0) states for the grammar. Links +** are added to between some states so that the LR(1) follow sets +** can be computed later. +*/ +PRIVATE struct state *getstate(struct lemon *); /* forward reference */ +void FindStates(struct lemon *lemp) +{ + struct symbol *sp; + struct rule *rp; + + Configlist_init(); + + /* Find the start symbol */ + if( lemp->start ){ + sp = Symbol_find(lemp->start); + if( sp==0 ){ + ErrorMsg(lemp->filename,0, + "The specified start symbol \"%s\" is not " + "in a nonterminal of the grammar. \"%s\" will be used as the start " + "symbol instead.",lemp->start,lemp->startRule->lhs->name); + lemp->errorcnt++; + sp = lemp->startRule->lhs; + } + }else if( lemp->startRule ){ + sp = lemp->startRule->lhs; + }else{ + ErrorMsg(lemp->filename,0,"Internal error - no start rule\n"); + exit(1); + } + + /* Make sure the start symbol doesn't occur on the right-hand side of + ** any rule. Report an error if it does. (YACC would generate a new + ** start symbol in this case.) */ + for(rp=lemp->rule; rp; rp=rp->next){ + int i; + for(i=0; i<rp->nrhs; i++){ + if( rp->rhs[i]==sp ){ /* FIX ME: Deal with multiterminals */ + ErrorMsg(lemp->filename,0, + "The start symbol \"%s\" occurs on the " + "right-hand side of a rule. This will result in a parser which " + "does not work properly.",sp->name); + lemp->errorcnt++; + } + } + } + + /* The basis configuration set for the first state + ** is all rules which have the start symbol as their + ** left-hand side */ + for(rp=sp->rule; rp; rp=rp->nextlhs){ + struct config *newcfp; + rp->lhsStart = 1; + newcfp = Configlist_addbasis(rp,0); + SetAdd(newcfp->fws,0); + } + + /* Compute the first state. All other states will be + ** computed automatically during the computation of the first one. + ** The returned pointer to the first state is not used. */ + (void)getstate(lemp); + return; +} + +/* Return a pointer to a state which is described by the configuration +** list which has been built from calls to Configlist_add. +*/ +PRIVATE void buildshifts(struct lemon *, struct state *); /* Forwd ref */ +PRIVATE struct state *getstate(struct lemon *lemp) +{ + struct config *cfp, *bp; + struct state *stp; + + /* Extract the sorted basis of the new state. The basis was constructed + ** by prior calls to "Configlist_addbasis()". */ + Configlist_sortbasis(); + bp = Configlist_basis(); + + /* Get a state with the same basis */ + stp = State_find(bp); + if( stp ){ + /* A state with the same basis already exists! Copy all the follow-set + ** propagation links from the state under construction into the + ** preexisting state, then return a pointer to the preexisting state */ + struct config *x, *y; + for(x=bp, y=stp->bp; x && y; x=x->bp, y=y->bp){ + Plink_copy(&y->bplp,x->bplp); + Plink_delete(x->fplp); + x->fplp = x->bplp = 0; + } + cfp = Configlist_return(); + Configlist_eat(cfp); + }else{ + /* This really is a new state. Construct all the details */ + Configlist_closure(lemp); /* Compute the configuration closure */ + Configlist_sort(); /* Sort the configuration closure */ + cfp = Configlist_return(); /* Get a pointer to the config list */ + stp = State_new(); /* A new state structure */ + MemoryCheck(stp); + stp->bp = bp; /* Remember the configuration basis */ + stp->cfp = cfp; /* Remember the configuration closure */ + stp->statenum = lemp->nstate++; /* Every state gets a sequence number */ + stp->ap = 0; /* No actions, yet. */ + State_insert(stp,stp->bp); /* Add to the state table */ + buildshifts(lemp,stp); /* Recursively compute successor states */ + } + return stp; +} + +/* +** Return true if two symbols are the same. +*/ +int same_symbol(struct symbol *a, struct symbol *b) +{ + int i; + if( a==b ) return 1; + if( a->type!=MULTITERMINAL ) return 0; + if( b->type!=MULTITERMINAL ) return 0; + if( a->nsubsym!=b->nsubsym ) return 0; + for(i=0; i<a->nsubsym; i++){ + if( a->subsym[i]!=b->subsym[i] ) return 0; + } + return 1; +} + +/* Construct all successor states to the given state. A "successor" +** state is any state which can be reached by a shift action. +*/ +PRIVATE void buildshifts(struct lemon *lemp, struct state *stp) +{ + struct config *cfp; /* For looping thru the config closure of "stp" */ + struct config *bcfp; /* For the inner loop on config closure of "stp" */ + struct config *newcfg; /* */ + struct symbol *sp; /* Symbol following the dot in configuration "cfp" */ + struct symbol *bsp; /* Symbol following the dot in configuration "bcfp" */ + struct state *newstp; /* A pointer to a successor state */ + + /* Each configuration becomes complete after it contributes to a successor + ** state. Initially, all configurations are incomplete */ + for(cfp=stp->cfp; cfp; cfp=cfp->next) cfp->status = INCOMPLETE; + + /* Loop through all configurations of the state "stp" */ + for(cfp=stp->cfp; cfp; cfp=cfp->next){ + if( cfp->status==COMPLETE ) continue; /* Already used by inner loop */ + if( cfp->dot>=cfp->rp->nrhs ) continue; /* Can't shift this config */ + Configlist_reset(); /* Reset the new config set */ + sp = cfp->rp->rhs[cfp->dot]; /* Symbol after the dot */ + + /* For every configuration in the state "stp" which has the symbol "sp" + ** following its dot, add the same configuration to the basis set under + ** construction but with the dot shifted one symbol to the right. */ + for(bcfp=cfp; bcfp; bcfp=bcfp->next){ + if( bcfp->status==COMPLETE ) continue; /* Already used */ + if( bcfp->dot>=bcfp->rp->nrhs ) continue; /* Can't shift this one */ + bsp = bcfp->rp->rhs[bcfp->dot]; /* Get symbol after dot */ + if( !same_symbol(bsp,sp) ) continue; /* Must be same as for "cfp" */ + bcfp->status = COMPLETE; /* Mark this config as used */ + newcfg = Configlist_addbasis(bcfp->rp,bcfp->dot+1); + Plink_add(&newcfg->bplp,bcfp); + } + + /* Get a pointer to the state described by the basis configuration set + ** constructed in the preceding loop */ + newstp = getstate(lemp); + + /* The state "newstp" is reached from the state "stp" by a shift action + ** on the symbol "sp" */ + if( sp->type==MULTITERMINAL ){ + int i; + for(i=0; i<sp->nsubsym; i++){ + Action_add(&stp->ap,SHIFT,sp->subsym[i],(char*)newstp); + } + }else{ + Action_add(&stp->ap,SHIFT,sp,(char *)newstp); + } + } +} + +/* +** Construct the propagation links +*/ +void FindLinks(struct lemon *lemp) +{ + int i; + struct config *cfp, *other; + struct state *stp; + struct plink *plp; + + /* Housekeeping detail: + ** Add to every propagate link a pointer back to the state to + ** which the link is attached. */ + for(i=0; i<lemp->nstate; i++){ + stp = lemp->sorted[i]; + for(cfp=stp?stp->cfp:0; cfp; cfp=cfp->next){ + cfp->stp = stp; + } + } + + /* Convert all backlinks into forward links. Only the forward + ** links are used in the follow-set computation. */ + for(i=0; i<lemp->nstate; i++){ + stp = lemp->sorted[i]; + for(cfp=stp?stp->cfp:0; cfp; cfp=cfp->next){ + for(plp=cfp->bplp; plp; plp=plp->next){ + other = plp->cfp; + Plink_add(&other->fplp,cfp); + } + } + } +} + +/* Compute all followsets. +** +** A followset is the set of all symbols which can come immediately +** after a configuration. +*/ +void FindFollowSets(struct lemon *lemp) +{ + int i; + struct config *cfp; + struct plink *plp; + int progress; + int change; + + for(i=0; i<lemp->nstate; i++){ + assert( lemp->sorted[i]!=0 ); + for(cfp=lemp->sorted[i]->cfp; cfp; cfp=cfp->next){ + cfp->status = INCOMPLETE; + } + } + + do{ + progress = 0; + for(i=0; i<lemp->nstate; i++){ + assert( lemp->sorted[i]!=0 ); + for(cfp=lemp->sorted[i]->cfp; cfp; cfp=cfp->next){ + if( cfp->status==COMPLETE ) continue; + for(plp=cfp->fplp; plp; plp=plp->next){ + change = SetUnion(plp->cfp->fws,cfp->fws); + if( change ){ + plp->cfp->status = INCOMPLETE; + progress = 1; + } + } + cfp->status = COMPLETE; + } + } + }while( progress ); +} + +static int resolve_conflict(struct action *,struct action *); + +/* Compute the reduce actions, and resolve conflicts. +*/ +void FindActions(struct lemon *lemp) +{ + int i,j; + struct config *cfp; + struct state *stp; + struct symbol *sp; + struct rule *rp; + + /* Add all of the reduce actions + ** A reduce action is added for each element of the followset of + ** a configuration which has its dot at the extreme right. + */ + for(i=0; i<lemp->nstate; i++){ /* Loop over all states */ + stp = lemp->sorted[i]; + for(cfp=stp->cfp; cfp; cfp=cfp->next){ /* Loop over all configurations */ + if( cfp->rp->nrhs==cfp->dot ){ /* Is dot at extreme right? */ + for(j=0; j<lemp->nterminal; j++){ + if( SetFind(cfp->fws,j) ){ + /* Add a reduce action to the state "stp" which will reduce by the + ** rule "cfp->rp" if the lookahead symbol is "lemp->symbols[j]" */ + Action_add(&stp->ap,REDUCE,lemp->symbols[j],(char *)cfp->rp); + } + } + } + } + } + + /* Add the accepting token */ + if( lemp->start ){ + sp = Symbol_find(lemp->start); + if( sp==0 ){ + if( lemp->startRule==0 ){ + fprintf(stderr, "internal error on source line %d: no start rule\n", + __LINE__); + exit(1); + } + sp = lemp->startRule->lhs; + } + }else{ + sp = lemp->startRule->lhs; + } + /* Add to the first state (which is always the starting state of the + ** finite state machine) an action to ACCEPT if the lookahead is the + ** start nonterminal. */ + Action_add(&lemp->sorted[0]->ap,ACCEPT,sp,0); + + /* Resolve conflicts */ + for(i=0; i<lemp->nstate; i++){ + struct action *ap, *nap; + stp = lemp->sorted[i]; + /* assert( stp->ap ); */ + stp->ap = Action_sort(stp->ap); + for(ap=stp->ap; ap && ap->next; ap=ap->next){ + for(nap=ap->next; nap && nap->sp==ap->sp; nap=nap->next){ + /* The two actions "ap" and "nap" have the same lookahead. + ** Figure out which one should be used */ + lemp->nconflict += resolve_conflict(ap,nap); + } + } + } + + /* Report an error for each rule that can never be reduced. */ + for(rp=lemp->rule; rp; rp=rp->next) rp->canReduce = LEMON_FALSE; + for(i=0; i<lemp->nstate; i++){ + struct action *ap; + for(ap=lemp->sorted[i]->ap; ap; ap=ap->next){ + if( ap->type==REDUCE ) ap->x.rp->canReduce = LEMON_TRUE; + } + } + for(rp=lemp->rule; rp; rp=rp->next){ + if( rp->canReduce ) continue; + ErrorMsg(lemp->filename,rp->ruleline,"This rule can not be reduced.\n"); + lemp->errorcnt++; + } +} + +/* Resolve a conflict between the two given actions. If the +** conflict can't be resolved, return non-zero. +** +** NO LONGER TRUE: +** To resolve a conflict, first look to see if either action +** is on an error rule. In that case, take the action which +** is not associated with the error rule. If neither or both +** actions are associated with an error rule, then try to +** use precedence to resolve the conflict. +** +** If either action is a SHIFT, then it must be apx. This +** function won't work if apx->type==REDUCE and apy->type==SHIFT. +*/ +static int resolve_conflict( + struct action *apx, + struct action *apy +){ + struct symbol *spx, *spy; + int errcnt = 0; + assert( apx->sp==apy->sp ); /* Otherwise there would be no conflict */ + if( apx->type==SHIFT && apy->type==SHIFT ){ + apy->type = SSCONFLICT; + errcnt++; + } + if( apx->type==SHIFT && apy->type==REDUCE ){ + spx = apx->sp; + spy = apy->x.rp->precsym; + if( spy==0 || spx->prec<0 || spy->prec<0 ){ + /* Not enough precedence information. */ + apy->type = SRCONFLICT; + errcnt++; + }else if( spx->prec>spy->prec ){ /* higher precedence wins */ + apy->type = RD_RESOLVED; + }else if( spx->prec<spy->prec ){ + apx->type = SH_RESOLVED; + }else if( spx->prec==spy->prec && spx->assoc==RIGHT ){ /* Use operator */ + apy->type = RD_RESOLVED; /* associativity */ + }else if( spx->prec==spy->prec && spx->assoc==LEFT ){ /* to break tie */ + apx->type = SH_RESOLVED; + }else{ + assert( spx->prec==spy->prec && spx->assoc==NONE ); + apx->type = ERROR; + } + }else if( apx->type==REDUCE && apy->type==REDUCE ){ + spx = apx->x.rp->precsym; + spy = apy->x.rp->precsym; + if( spx==0 || spy==0 || spx->prec<0 || + spy->prec<0 || spx->prec==spy->prec ){ + apy->type = RRCONFLICT; + errcnt++; + }else if( spx->prec>spy->prec ){ + apy->type = RD_RESOLVED; + }else if( spx->prec<spy->prec ){ + apx->type = RD_RESOLVED; + } + }else{ + assert( + apx->type==SH_RESOLVED || + apx->type==RD_RESOLVED || + apx->type==SSCONFLICT || + apx->type==SRCONFLICT || + apx->type==RRCONFLICT || + apy->type==SH_RESOLVED || + apy->type==RD_RESOLVED || + apy->type==SSCONFLICT || + apy->type==SRCONFLICT || + apy->type==RRCONFLICT + ); + /* The REDUCE/SHIFT case cannot happen because SHIFTs come before + ** REDUCEs on the list. If we reach this point it must be because + ** the parser conflict had already been resolved. */ + } + return errcnt; +} +/********************* From the file "configlist.c" *************************/ +/* +** Routines to processing a configuration list and building a state +** in the LEMON parser generator. +*/ + +static struct config *freelist = 0; /* List of free configurations */ +static struct config *current = 0; /* Top of list of configurations */ +static struct config **currentend = 0; /* Last on list of configs */ +static struct config *basis = 0; /* Top of list of basis configs */ +static struct config **basisend = 0; /* End of list of basis configs */ + +/* Return a pointer to a new configuration */ +PRIVATE struct config *newconfig(void){ + return (struct config*)calloc(1, sizeof(struct config)); +} + +/* The configuration "old" is no longer used */ +PRIVATE void deleteconfig(struct config *old) +{ + old->next = freelist; + freelist = old; +} + +/* Initialized the configuration list builder */ +void Configlist_init(void){ + current = 0; + currentend = ¤t; + basis = 0; + basisend = &basis; + Configtable_init(); + return; +} + +/* Initialized the configuration list builder */ +void Configlist_reset(void){ + current = 0; + currentend = ¤t; + basis = 0; + basisend = &basis; + Configtable_clear(0); + return; +} + +/* Add another configuration to the configuration list */ +struct config *Configlist_add( + struct rule *rp, /* The rule */ + int dot /* Index into the RHS of the rule where the dot goes */ +){ + struct config *cfp, model; + + assert( currentend!=0 ); + model.rp = rp; + model.dot = dot; + cfp = Configtable_find(&model); + if( cfp==0 ){ + cfp = newconfig(); + cfp->rp = rp; + cfp->dot = dot; + cfp->fws = SetNew(); + cfp->stp = 0; + cfp->fplp = cfp->bplp = 0; + cfp->next = 0; + cfp->bp = 0; + *currentend = cfp; + currentend = &cfp->next; + Configtable_insert(cfp); + } + return cfp; +} + +/* Add a basis configuration to the configuration list */ +struct config *Configlist_addbasis(struct rule *rp, int dot) +{ + struct config *cfp, model; + + assert( basisend!=0 ); + assert( currentend!=0 ); + model.rp = rp; + model.dot = dot; + cfp = Configtable_find(&model); + if( cfp==0 ){ + cfp = newconfig(); + cfp->rp = rp; + cfp->dot = dot; + cfp->fws = SetNew(); + cfp->stp = 0; + cfp->fplp = cfp->bplp = 0; + cfp->next = 0; + cfp->bp = 0; + *currentend = cfp; + currentend = &cfp->next; + *basisend = cfp; + basisend = &cfp->bp; + Configtable_insert(cfp); + } + return cfp; +} + +/* Compute the closure of the configuration list */ +void Configlist_closure(struct lemon *lemp) +{ + struct config *cfp, *newcfp; + struct rule *rp, *newrp; + struct symbol *sp, *xsp; + int i, dot; + + assert( currentend!=0 ); + for(cfp=current; cfp; cfp=cfp->next){ + rp = cfp->rp; + dot = cfp->dot; + if( dot>=rp->nrhs ) continue; + sp = rp->rhs[dot]; + if( sp->type==NONTERMINAL ){ + if( sp->rule==0 && sp!=lemp->errsym ){ + ErrorMsg(lemp->filename,rp->line,"Nonterminal \"%s\" has no rules.", + sp->name); + lemp->errorcnt++; + } + for(newrp=sp->rule; newrp; newrp=newrp->nextlhs){ + newcfp = Configlist_add(newrp,0); + for(i=dot+1; i<rp->nrhs; i++){ + xsp = rp->rhs[i]; + if( xsp->type==TERMINAL ){ + SetAdd(newcfp->fws,xsp->index); + break; + }else if( xsp->type==MULTITERMINAL ){ + int k; + for(k=0; k<xsp->nsubsym; k++){ + SetAdd(newcfp->fws, xsp->subsym[k]->index); + } + break; + }else{ + SetUnion(newcfp->fws,xsp->firstset); + if( xsp->lambda==LEMON_FALSE ) break; + } + } + if( i==rp->nrhs ) Plink_add(&cfp->fplp,newcfp); + } + } + } + return; +} + +/* Sort the configuration list */ +void Configlist_sort(void){ + current = (struct config*)msort((char*)current,(char**)&(current->next), + Configcmp); + currentend = 0; + return; +} + +/* Sort the basis configuration list */ +void Configlist_sortbasis(void){ + basis = (struct config*)msort((char*)current,(char**)&(current->bp), + Configcmp); + basisend = 0; + return; +} + +/* Return a pointer to the head of the configuration list and +** reset the list */ +struct config *Configlist_return(void){ + struct config *old; + old = current; + current = 0; + currentend = 0; + return old; +} + +/* Return a pointer to the head of the configuration list and +** reset the list */ +struct config *Configlist_basis(void){ + struct config *old; + old = basis; + basis = 0; + basisend = 0; + return old; +} + +/* Free all elements of the given configuration list */ +void Configlist_eat(struct config *cfp) +{ + struct config *nextcfp; + for(; cfp; cfp=nextcfp){ + nextcfp = cfp->next; + assert( cfp->fplp==0 ); + assert( cfp->bplp==0 ); + if( cfp->fws ) SetFree(cfp->fws); + deleteconfig(cfp); + } + return; +} +/***************** From the file "error.c" *********************************/ +/* +** Code for printing error message. +*/ + +void ErrorMsg(const char *filename, int lineno, const char *format, ...){ + va_list ap; + fprintf(stderr, "%s:%d: ", filename, lineno); + va_start(ap, format); + vfprintf(stderr,format,ap); + va_end(ap); + fprintf(stderr, "\n"); +} +/**************** From the file "main.c" ************************************/ +/* +** Main program file for the LEMON parser generator. +*/ + +/* Report an out-of-memory condition and abort. This function +** is used mostly by the "MemoryCheck" macro in struct.h +*/ +void memory_error(void){ + fprintf(stderr,"Out of memory. Aborting...\n"); + exit(1); +} + +static int nDefine = 0; /* Number of -D options on the command line */ +static int nDefineUsed = 0; /* Number of -D options actually used */ +static char **azDefine = 0; /* Name of the -D macros */ +static char *bDefineUsed = 0; /* True for every -D macro actually used */ + +/* This routine is called with the argument to each -D command-line option. +** Add the macro defined to the azDefine array. +*/ +static void handle_D_option(char *z){ + char **paz; + nDefine++; + azDefine = (char **) realloc(azDefine, sizeof(azDefine[0])*nDefine); + if( azDefine==0 ){ + fprintf(stderr,"out of memory\n"); + exit(1); + } + bDefineUsed = (char*)realloc(bDefineUsed, nDefine); + if( bDefineUsed==0 ){ + fprintf(stderr,"out of memory\n"); + exit(1); + } + bDefineUsed[nDefine-1] = 0; + paz = &azDefine[nDefine-1]; + *paz = (char *) malloc( lemonStrlen(z)+1 ); + if( *paz==0 ){ + fprintf(stderr,"out of memory\n"); + exit(1); + } + lemon_strcpy(*paz, z); + for(z=*paz; *z && *z!='='; z++){} + *z = 0; +} + +/* Rember the name of the output directory +*/ +static char *outputDir = NULL; +static void handle_d_option(char *z){ + outputDir = (char *) malloc( lemonStrlen(z)+1 ); + if( outputDir==0 ){ + fprintf(stderr,"out of memory\n"); + exit(1); + } + lemon_strcpy(outputDir, z); +} + +static char *user_templatename = NULL; +static void handle_T_option(char *z){ + user_templatename = (char *) malloc( lemonStrlen(z)+1 ); + if( user_templatename==0 ){ + memory_error(); + } + lemon_strcpy(user_templatename, z); +} + +/* Merge together to lists of rules ordered by rule.iRule */ +static struct rule *Rule_merge(struct rule *pA, struct rule *pB){ + struct rule *pFirst = 0; + struct rule **ppPrev = &pFirst; + while( pA && pB ){ + if( pA->iRule<pB->iRule ){ + *ppPrev = pA; + ppPrev = &pA->next; + pA = pA->next; + }else{ + *ppPrev = pB; + ppPrev = &pB->next; + pB = pB->next; + } + } + if( pA ){ + *ppPrev = pA; + }else{ + *ppPrev = pB; + } + return pFirst; +} + +/* +** Sort a list of rules in order of increasing iRule value +*/ +static struct rule *Rule_sort(struct rule *rp){ + unsigned int i; + struct rule *pNext; + struct rule *x[32]; + memset(x, 0, sizeof(x)); + while( rp ){ + pNext = rp->next; + rp->next = 0; + for(i=0; i<sizeof(x)/sizeof(x[0])-1 && x[i]; i++){ + rp = Rule_merge(x[i], rp); + x[i] = 0; + } + x[i] = rp; + rp = pNext; + } + rp = 0; + for(i=0; i<sizeof(x)/sizeof(x[0]); i++){ + rp = Rule_merge(x[i], rp); + } + return rp; +} + +/* forward reference */ +static const char *minimum_size_type(int lwr, int upr, int *pnByte); + +/* Print a single line of the "Parser Stats" output +*/ +static void stats_line(const char *zLabel, int iValue){ + int nLabel = lemonStrlen(zLabel); + printf(" %s%.*s %5d\n", zLabel, + 35-nLabel, "................................", + iValue); +} + +/* The main program. Parse the command line and do it... */ +int main(int argc, char **argv){ + static int version = 0; + static int rpflag = 0; + static int basisflag = 0; + static int compress = 0; + static int quiet = 0; + static int statistics = 0; + static int mhflag = 0; + static int nolinenosflag = 0; + static int noResort = 0; + static int sqlFlag = 0; + static int printPP = 0; + + static struct s_options options[] = { + {OPT_FLAG, "b", (char*)&basisflag, "Print only the basis in report."}, + {OPT_FLAG, "c", (char*)&compress, "Don't compress the action table."}, + {OPT_FSTR, "d", (char*)&handle_d_option, "Output directory. Default '.'"}, + {OPT_FSTR, "D", (char*)handle_D_option, "Define an %ifdef macro."}, + {OPT_FLAG, "E", (char*)&printPP, "Print input file after preprocessing."}, + {OPT_FSTR, "f", 0, "Ignored. (Placeholder for -f compiler options.)"}, + {OPT_FLAG, "g", (char*)&rpflag, "Print grammar without actions."}, + {OPT_FSTR, "I", 0, "Ignored. (Placeholder for '-I' compiler options.)"}, + {OPT_FLAG, "m", (char*)&mhflag, "Output a makeheaders compatible file."}, + {OPT_FLAG, "l", (char*)&nolinenosflag, "Do not print #line statements."}, + {OPT_FSTR, "O", 0, "Ignored. (Placeholder for '-O' compiler options.)"}, + {OPT_FLAG, "p", (char*)&showPrecedenceConflict, + "Show conflicts resolved by precedence rules"}, + {OPT_FLAG, "q", (char*)&quiet, "(Quiet) Don't print the report file."}, + {OPT_FLAG, "r", (char*)&noResort, "Do not sort or renumber states"}, + {OPT_FLAG, "s", (char*)&statistics, + "Print parser stats to standard output."}, + {OPT_FLAG, "S", (char*)&sqlFlag, + "Generate the *.sql file describing the parser tables."}, + {OPT_FLAG, "x", (char*)&version, "Print the version number."}, + {OPT_FSTR, "T", (char*)handle_T_option, "Specify a template file."}, + {OPT_FSTR, "W", 0, "Ignored. (Placeholder for '-W' compiler options.)"}, + {OPT_FLAG,0,0,0} + }; + int i; + int exitcode; + struct lemon lem; + struct rule *rp; + + OptInit(argv,options,stderr); + if( version ){ + printf("Lemon version 1.0\n"); + exit(0); + } + if( OptNArgs()!=1 ){ + fprintf(stderr,"Exactly one filename argument is required.\n"); + exit(1); + } + memset(&lem, 0, sizeof(lem)); + lem.errorcnt = 0; + + /* Initialize the machine */ + Strsafe_init(); + Symbol_init(); + State_init(); + lem.argv = argv; + lem.argc = argc; + lem.filename = OptArg(0); + lem.basisflag = basisflag; + lem.nolinenosflag = nolinenosflag; + lem.printPreprocessed = printPP; + Symbol_new("$"); + + /* Parse the input file */ + Parse(&lem); + if( lem.printPreprocessed || lem.errorcnt ) exit(lem.errorcnt); + if( lem.nrule==0 ){ + fprintf(stderr,"Empty grammar.\n"); + exit(1); + } + lem.errsym = Symbol_find("error"); + + /* Count and index the symbols of the grammar */ + Symbol_new("{default}"); + lem.nsymbol = Symbol_count(); + lem.symbols = Symbol_arrayof(); + for(i=0; i<lem.nsymbol; i++) lem.symbols[i]->index = i; + qsort(lem.symbols,lem.nsymbol,sizeof(struct symbol*), Symbolcmpp); + for(i=0; i<lem.nsymbol; i++) lem.symbols[i]->index = i; + while( lem.symbols[i-1]->type==MULTITERMINAL ){ i--; } + assert( strcmp(lem.symbols[i-1]->name,"{default}")==0 ); + lem.nsymbol = i - 1; + for(i=1; ISUPPER(lem.symbols[i]->name[0]); i++); + lem.nterminal = i; + + /* Assign sequential rule numbers. Start with 0. Put rules that have no + ** reduce action C-code associated with them last, so that the switch() + ** statement that selects reduction actions will have a smaller jump table. + */ + for(i=0, rp=lem.rule; rp; rp=rp->next){ + rp->iRule = rp->code ? i++ : -1; + } + lem.nruleWithAction = i; + for(rp=lem.rule; rp; rp=rp->next){ + if( rp->iRule<0 ) rp->iRule = i++; + } + lem.startRule = lem.rule; + lem.rule = Rule_sort(lem.rule); + + /* Generate a reprint of the grammar, if requested on the command line */ + if( rpflag ){ + Reprint(&lem); + }else{ + /* Initialize the size for all follow and first sets */ + SetSize(lem.nterminal+1); + + /* Find the precedence for every production rule (that has one) */ + FindRulePrecedences(&lem); + + /* Compute the lambda-nonterminals and the first-sets for every + ** nonterminal */ + FindFirstSets(&lem); + + /* Compute all LR(0) states. Also record follow-set propagation + ** links so that the follow-set can be computed later */ + lem.nstate = 0; + FindStates(&lem); + lem.sorted = State_arrayof(); + + /* Tie up loose ends on the propagation links */ + FindLinks(&lem); + + /* Compute the follow set of every reducible configuration */ + FindFollowSets(&lem); + + /* Compute the action tables */ + FindActions(&lem); + + /* Compress the action tables */ + if( compress==0 ) CompressTables(&lem); + + /* Reorder and renumber the states so that states with fewer choices + ** occur at the end. This is an optimization that helps make the + ** generated parser tables smaller. */ + if( noResort==0 ) ResortStates(&lem); + + /* Generate a report of the parser generated. (the "y.output" file) */ + if( !quiet ) ReportOutput(&lem); + + /* Generate the source code for the parser */ + ReportTable(&lem, mhflag, sqlFlag); + + /* Produce a header file for use by the scanner. (This step is + ** omitted if the "-m" option is used because makeheaders will + ** generate the file for us.) */ + if( !mhflag ) ReportHeader(&lem); + } + if( statistics ){ + printf("Parser statistics:\n"); + stats_line("terminal symbols", lem.nterminal); + stats_line("non-terminal symbols", lem.nsymbol - lem.nterminal); + stats_line("total symbols", lem.nsymbol); + stats_line("rules", lem.nrule); + stats_line("states", lem.nxstate); + stats_line("conflicts", lem.nconflict); + stats_line("action table entries", lem.nactiontab); + stats_line("lookahead table entries", lem.nlookaheadtab); + stats_line("total table size (bytes)", lem.tablesize); + } + if( lem.nconflict > 0 ){ + fprintf(stderr,"%d parsing conflicts.\n",lem.nconflict); + } + + /* return 0 on success, 1 on failure. */ + exitcode = ((lem.errorcnt > 0) || (lem.nconflict > 0)) ? 1 : 0; + exit(exitcode); + return (exitcode); +} +/******************** From the file "msort.c" *******************************/ +/* +** A generic merge-sort program. +** +** USAGE: +** Let "ptr" be a pointer to some structure which is at the head of +** a null-terminated list. Then to sort the list call: +** +** ptr = msort(ptr,&(ptr->next),cmpfnc); +** +** In the above, "cmpfnc" is a pointer to a function which compares +** two instances of the structure and returns an integer, as in +** strcmp. The second argument is a pointer to the pointer to the +** second element of the linked list. This address is used to compute +** the offset to the "next" field within the structure. The offset to +** the "next" field must be constant for all structures in the list. +** +** The function returns a new pointer which is the head of the list +** after sorting. +** +** ALGORITHM: +** Merge-sort. +*/ + +/* +** Return a pointer to the next structure in the linked list. +*/ +#define NEXT(A) (*(char**)(((char*)A)+offset)) + +/* +** Inputs: +** a: A sorted, null-terminated linked list. (May be null). +** b: A sorted, null-terminated linked list. (May be null). +** cmp: A pointer to the comparison function. +** offset: Offset in the structure to the "next" field. +** +** Return Value: +** A pointer to the head of a sorted list containing the elements +** of both a and b. +** +** Side effects: +** The "next" pointers for elements in the lists a and b are +** changed. +*/ +static char *merge( + char *a, + char *b, + int (*cmp)(const char*,const char*), + int offset +){ + char *ptr, *head; + + if( a==0 ){ + head = b; + }else if( b==0 ){ + head = a; + }else{ + if( (*cmp)(a,b)<=0 ){ + ptr = a; + a = NEXT(a); + }else{ + ptr = b; + b = NEXT(b); + } + head = ptr; + while( a && b ){ + if( (*cmp)(a,b)<=0 ){ + NEXT(ptr) = a; + ptr = a; + a = NEXT(a); + }else{ + NEXT(ptr) = b; + ptr = b; + b = NEXT(b); + } + } + if( a ) NEXT(ptr) = a; + else NEXT(ptr) = b; + } + return head; +} + +/* +** Inputs: +** list: Pointer to a singly-linked list of structures. +** next: Pointer to pointer to the second element of the list. +** cmp: A comparison function. +** +** Return Value: +** A pointer to the head of a sorted list containing the elements +** originally in list. +** +** Side effects: +** The "next" pointers for elements in list are changed. +*/ +#define LISTSIZE 30 +static char *msort( + char *list, + char **next, + int (*cmp)(const char*,const char*) +){ + unsigned long offset; + char *ep; + char *set[LISTSIZE]; + int i; + offset = (unsigned long)((char*)next - (char*)list); + for(i=0; i<LISTSIZE; i++) set[i] = 0; + while( list ){ + ep = list; + list = NEXT(list); + NEXT(ep) = 0; + for(i=0; i<LISTSIZE-1 && set[i]!=0; i++){ + ep = merge(ep,set[i],cmp,offset); + set[i] = 0; + } + set[i] = ep; + } + ep = 0; + for(i=0; i<LISTSIZE; i++) if( set[i] ) ep = merge(set[i],ep,cmp,offset); + return ep; +} +/************************ From the file "option.c" **************************/ +static char **g_argv; +static struct s_options *op; +static FILE *errstream; + +#define ISOPT(X) ((X)[0]=='-'||(X)[0]=='+'||strchr((X),'=')!=0) + +/* +** Print the command line with a carrot pointing to the k-th character +** of the n-th field. +*/ +static void errline(int n, int k, FILE *err) +{ + int spcnt, i; + if( g_argv[0] ){ + fprintf(err,"%s",g_argv[0]); + spcnt = lemonStrlen(g_argv[0]) + 1; + }else{ + spcnt = 0; + } + for(i=1; i<n && g_argv[i]; i++){ + fprintf(err," %s",g_argv[i]); + spcnt += lemonStrlen(g_argv[i])+1; + } + spcnt += k; + for(; g_argv[i]; i++) fprintf(err," %s",g_argv[i]); + if( spcnt<20 ){ + fprintf(err,"\n%*s^-- here\n",spcnt,""); + }else{ + fprintf(err,"\n%*shere --^\n",spcnt-7,""); + } +} + +/* +** Return the index of the N-th non-switch argument. Return -1 +** if N is out of range. +*/ +static int argindex(int n) +{ + int i; + int dashdash = 0; + if( g_argv!=0 && *g_argv!=0 ){ + for(i=1; g_argv[i]; i++){ + if( dashdash || !ISOPT(g_argv[i]) ){ + if( n==0 ) return i; + n--; + } + if( strcmp(g_argv[i],"--")==0 ) dashdash = 1; + } + } + return -1; +} + +static char emsg[] = "Command line syntax error: "; + +/* +** Process a flag command line argument. +*/ +static int handleflags(int i, FILE *err) +{ + int v; + int errcnt = 0; + int j; + for(j=0; op[j].label; j++){ + if( strncmp(&g_argv[i][1],op[j].label,lemonStrlen(op[j].label))==0 ) break; + } + v = g_argv[i][0]=='-' ? 1 : 0; + if( op[j].label==0 ){ + if( err ){ + fprintf(err,"%sundefined option.\n",emsg); + errline(i,1,err); + } + errcnt++; + }else if( op[j].arg==0 ){ + /* Ignore this option */ + }else if( op[j].type==OPT_FLAG ){ + *((int*)op[j].arg) = v; + }else if( op[j].type==OPT_FFLAG ){ + (*(void(*)(int))(op[j].arg))(v); + }else if( op[j].type==OPT_FSTR ){ + (*(void(*)(char *))(op[j].arg))(&g_argv[i][2]); + }else{ + if( err ){ + fprintf(err,"%smissing argument on switch.\n",emsg); + errline(i,1,err); + } + errcnt++; + } + return errcnt; +} + +/* +** Process a command line switch which has an argument. +*/ +static int handleswitch(int i, FILE *err) +{ + int lv = 0; + double dv = 0.0; + char *sv = 0, *end; + char *cp; + int j; + int errcnt = 0; + cp = strchr(g_argv[i],'='); + assert( cp!=0 ); + *cp = 0; + for(j=0; op[j].label; j++){ + if( strcmp(g_argv[i],op[j].label)==0 ) break; + } + *cp = '='; + if( op[j].label==0 ){ + if( err ){ + fprintf(err,"%sundefined option.\n",emsg); + errline(i,0,err); + } + errcnt++; + }else{ + cp++; + switch( op[j].type ){ + case OPT_FLAG: + case OPT_FFLAG: + if( err ){ + fprintf(err,"%soption requires an argument.\n",emsg); + errline(i,0,err); + } + errcnt++; + break; + case OPT_DBL: + case OPT_FDBL: + dv = strtod(cp,&end); + if( *end ){ + if( err ){ + fprintf(err, + "%sillegal character in floating-point argument.\n",emsg); + errline(i,(int)((char*)end-(char*)g_argv[i]),err); + } + errcnt++; + } + break; + case OPT_INT: + case OPT_FINT: + lv = strtol(cp,&end,0); + if( *end ){ + if( err ){ + fprintf(err,"%sillegal character in integer argument.\n",emsg); + errline(i,(int)((char*)end-(char*)g_argv[i]),err); + } + errcnt++; + } + break; + case OPT_STR: + case OPT_FSTR: + sv = cp; + break; + } + switch( op[j].type ){ + case OPT_FLAG: + case OPT_FFLAG: + break; + case OPT_DBL: + *(double*)(op[j].arg) = dv; + break; + case OPT_FDBL: + (*(void(*)(double))(op[j].arg))(dv); + break; + case OPT_INT: + *(int*)(op[j].arg) = lv; + break; + case OPT_FINT: + (*(void(*)(int))(op[j].arg))((int)lv); + break; + case OPT_STR: + *(char**)(op[j].arg) = sv; + break; + case OPT_FSTR: + (*(void(*)(char *))(op[j].arg))(sv); + break; + } + } + return errcnt; +} + +int OptInit(char **a, struct s_options *o, FILE *err) +{ + int errcnt = 0; + g_argv = a; + op = o; + errstream = err; + if( g_argv && *g_argv && op ){ + int i; + for(i=1; g_argv[i]; i++){ + if( g_argv[i][0]=='+' || g_argv[i][0]=='-' ){ + errcnt += handleflags(i,err); + }else if( strchr(g_argv[i],'=') ){ + errcnt += handleswitch(i,err); + } + } + } + if( errcnt>0 ){ + fprintf(err,"Valid command line options for \"%s\" are:\n",*a); + OptPrint(); + exit(1); + } + return 0; +} + +int OptNArgs(void){ + int cnt = 0; + int dashdash = 0; + int i; + if( g_argv!=0 && g_argv[0]!=0 ){ + for(i=1; g_argv[i]; i++){ + if( dashdash || !ISOPT(g_argv[i]) ) cnt++; + if( strcmp(g_argv[i],"--")==0 ) dashdash = 1; + } + } + return cnt; +} + +char *OptArg(int n) +{ + int i; + i = argindex(n); + return i>=0 ? g_argv[i] : 0; +} + +void OptErr(int n) +{ + int i; + i = argindex(n); + if( i>=0 ) errline(i,0,errstream); +} + +void OptPrint(void){ + int i; + int max, len; + max = 0; + for(i=0; op[i].label; i++){ + len = lemonStrlen(op[i].label) + 1; + switch( op[i].type ){ + case OPT_FLAG: + case OPT_FFLAG: + break; + case OPT_INT: + case OPT_FINT: + len += 9; /* length of "<integer>" */ + break; + case OPT_DBL: + case OPT_FDBL: + len += 6; /* length of "<real>" */ + break; + case OPT_STR: + case OPT_FSTR: + len += 8; /* length of "<string>" */ + break; + } + if( len>max ) max = len; + } + for(i=0; op[i].label; i++){ + switch( op[i].type ){ + case OPT_FLAG: + case OPT_FFLAG: + fprintf(errstream," -%-*s %s\n",max,op[i].label,op[i].message); + break; + case OPT_INT: + case OPT_FINT: + fprintf(errstream," -%s<integer>%*s %s\n",op[i].label, + (int)(max-lemonStrlen(op[i].label)-9),"",op[i].message); + break; + case OPT_DBL: + case OPT_FDBL: + fprintf(errstream," -%s<real>%*s %s\n",op[i].label, + (int)(max-lemonStrlen(op[i].label)-6),"",op[i].message); + break; + case OPT_STR: + case OPT_FSTR: + fprintf(errstream," -%s<string>%*s %s\n",op[i].label, + (int)(max-lemonStrlen(op[i].label)-8),"",op[i].message); + break; + } + } +} +/*********************** From the file "parse.c" ****************************/ +/* +** Input file parser for the LEMON parser generator. +*/ + +/* The state of the parser */ +enum e_state { + INITIALIZE, + WAITING_FOR_DECL_OR_RULE, + WAITING_FOR_DECL_KEYWORD, + WAITING_FOR_DECL_ARG, + WAITING_FOR_PRECEDENCE_SYMBOL, + WAITING_FOR_ARROW, + IN_RHS, + LHS_ALIAS_1, + LHS_ALIAS_2, + LHS_ALIAS_3, + RHS_ALIAS_1, + RHS_ALIAS_2, + PRECEDENCE_MARK_1, + PRECEDENCE_MARK_2, + RESYNC_AFTER_RULE_ERROR, + RESYNC_AFTER_DECL_ERROR, + WAITING_FOR_DESTRUCTOR_SYMBOL, + WAITING_FOR_DATATYPE_SYMBOL, + WAITING_FOR_FALLBACK_ID, + WAITING_FOR_WILDCARD_ID, + WAITING_FOR_CLASS_ID, + WAITING_FOR_CLASS_TOKEN, + WAITING_FOR_TOKEN_NAME +}; +struct pstate { + char *filename; /* Name of the input file */ + int tokenlineno; /* Linenumber at which current token starts */ + int errorcnt; /* Number of errors so far */ + char *tokenstart; /* Text of current token */ + struct lemon *gp; /* Global state vector */ + enum e_state state; /* The state of the parser */ + struct symbol *fallback; /* The fallback token */ + struct symbol *tkclass; /* Token class symbol */ + struct symbol *lhs; /* Left-hand side of current rule */ + const char *lhsalias; /* Alias for the LHS */ + int nrhs; /* Number of right-hand side symbols seen */ + struct symbol *rhs[MAXRHS]; /* RHS symbols */ + const char *alias[MAXRHS]; /* Aliases for each RHS symbol (or NULL) */ + struct rule *prevrule; /* Previous rule parsed */ + const char *declkeyword; /* Keyword of a declaration */ + char **declargslot; /* Where the declaration argument should be put */ + int insertLineMacro; /* Add #line before declaration insert */ + int *decllinenoslot; /* Where to write declaration line number */ + enum e_assoc declassoc; /* Assign this association to decl arguments */ + int preccounter; /* Assign this precedence to decl arguments */ + struct rule *firstrule; /* Pointer to first rule in the grammar */ + struct rule *lastrule; /* Pointer to the most recently parsed rule */ +}; + +/* Parse a single token */ +static void parseonetoken(struct pstate *psp) +{ + const char *x; + x = Strsafe(psp->tokenstart); /* Save the token permanently */ +#if 0 + printf("%s:%d: Token=[%s] state=%d\n",psp->filename,psp->tokenlineno, + x,psp->state); +#endif + switch( psp->state ){ + case INITIALIZE: + psp->prevrule = 0; + psp->preccounter = 0; + psp->firstrule = psp->lastrule = 0; + psp->gp->nrule = 0; + /* fall through */ + case WAITING_FOR_DECL_OR_RULE: + if( x[0]=='%' ){ + psp->state = WAITING_FOR_DECL_KEYWORD; + }else if( ISLOWER(x[0]) ){ + psp->lhs = Symbol_new(x); + psp->nrhs = 0; + psp->lhsalias = 0; + psp->state = WAITING_FOR_ARROW; + }else if( x[0]=='{' ){ + if( psp->prevrule==0 ){ + ErrorMsg(psp->filename,psp->tokenlineno, + "There is no prior rule upon which to attach the code " + "fragment which begins on this line."); + psp->errorcnt++; + }else if( psp->prevrule->code!=0 ){ + ErrorMsg(psp->filename,psp->tokenlineno, + "Code fragment beginning on this line is not the first " + "to follow the previous rule."); + psp->errorcnt++; + }else if( strcmp(x, "{NEVER-REDUCE")==0 ){ + psp->prevrule->neverReduce = 1; + }else{ + psp->prevrule->line = psp->tokenlineno; + psp->prevrule->code = &x[1]; + psp->prevrule->noCode = 0; + } + }else if( x[0]=='[' ){ + psp->state = PRECEDENCE_MARK_1; + }else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Token \"%s\" should be either \"%%\" or a nonterminal name.", + x); + psp->errorcnt++; + } + break; + case PRECEDENCE_MARK_1: + if( !ISUPPER(x[0]) ){ + ErrorMsg(psp->filename,psp->tokenlineno, + "The precedence symbol must be a terminal."); + psp->errorcnt++; + }else if( psp->prevrule==0 ){ + ErrorMsg(psp->filename,psp->tokenlineno, + "There is no prior rule to assign precedence \"[%s]\".",x); + psp->errorcnt++; + }else if( psp->prevrule->precsym!=0 ){ + ErrorMsg(psp->filename,psp->tokenlineno, + "Precedence mark on this line is not the first " + "to follow the previous rule."); + psp->errorcnt++; + }else{ + psp->prevrule->precsym = Symbol_new(x); + } + psp->state = PRECEDENCE_MARK_2; + break; + case PRECEDENCE_MARK_2: + if( x[0]!=']' ){ + ErrorMsg(psp->filename,psp->tokenlineno, + "Missing \"]\" on precedence mark."); + psp->errorcnt++; + } + psp->state = WAITING_FOR_DECL_OR_RULE; + break; + case WAITING_FOR_ARROW: + if( x[0]==':' && x[1]==':' && x[2]=='=' ){ + psp->state = IN_RHS; + }else if( x[0]=='(' ){ + psp->state = LHS_ALIAS_1; + }else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Expected to see a \":\" following the LHS symbol \"%s\".", + psp->lhs->name); + psp->errorcnt++; + psp->state = RESYNC_AFTER_RULE_ERROR; + } + break; + case LHS_ALIAS_1: + if( ISALPHA(x[0]) ){ + psp->lhsalias = x; + psp->state = LHS_ALIAS_2; + }else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "\"%s\" is not a valid alias for the LHS \"%s\"\n", + x,psp->lhs->name); + psp->errorcnt++; + psp->state = RESYNC_AFTER_RULE_ERROR; + } + break; + case LHS_ALIAS_2: + if( x[0]==')' ){ + psp->state = LHS_ALIAS_3; + }else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Missing \")\" following LHS alias name \"%s\".",psp->lhsalias); + psp->errorcnt++; + psp->state = RESYNC_AFTER_RULE_ERROR; + } + break; + case LHS_ALIAS_3: + if( x[0]==':' && x[1]==':' && x[2]=='=' ){ + psp->state = IN_RHS; + }else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Missing \"->\" following: \"%s(%s)\".", + psp->lhs->name,psp->lhsalias); + psp->errorcnt++; + psp->state = RESYNC_AFTER_RULE_ERROR; + } + break; + case IN_RHS: + if( x[0]=='.' ){ + struct rule *rp; + rp = (struct rule *)calloc( sizeof(struct rule) + + sizeof(struct symbol*)*psp->nrhs + sizeof(char*)*psp->nrhs, 1); + if( rp==0 ){ + ErrorMsg(psp->filename,psp->tokenlineno, + "Can't allocate enough memory for this rule."); + psp->errorcnt++; + psp->prevrule = 0; + }else{ + int i; + rp->ruleline = psp->tokenlineno; + rp->rhs = (struct symbol**)&rp[1]; + rp->rhsalias = (const char**)&(rp->rhs[psp->nrhs]); + for(i=0; i<psp->nrhs; i++){ + rp->rhs[i] = psp->rhs[i]; + rp->rhsalias[i] = psp->alias[i]; + if( rp->rhsalias[i]!=0 ){ rp->rhs[i]->bContent = 1; } + } + rp->lhs = psp->lhs; + rp->lhsalias = psp->lhsalias; + rp->nrhs = psp->nrhs; + rp->code = 0; + rp->noCode = 1; + rp->precsym = 0; + rp->index = psp->gp->nrule++; + rp->nextlhs = rp->lhs->rule; + rp->lhs->rule = rp; + rp->next = 0; + if( psp->firstrule==0 ){ + psp->firstrule = psp->lastrule = rp; + }else{ + psp->lastrule->next = rp; + psp->lastrule = rp; + } + psp->prevrule = rp; + } + psp->state = WAITING_FOR_DECL_OR_RULE; + }else if( ISALPHA(x[0]) ){ + if( psp->nrhs>=MAXRHS ){ + ErrorMsg(psp->filename,psp->tokenlineno, + "Too many symbols on RHS of rule beginning at \"%s\".", + x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_RULE_ERROR; + }else{ + psp->rhs[psp->nrhs] = Symbol_new(x); + psp->alias[psp->nrhs] = 0; + psp->nrhs++; + } + }else if( (x[0]=='|' || x[0]=='/') && psp->nrhs>0 && ISUPPER(x[1]) ){ + struct symbol *msp = psp->rhs[psp->nrhs-1]; + if( msp->type!=MULTITERMINAL ){ + struct symbol *origsp = msp; + msp = (struct symbol *) calloc(1,sizeof(*msp)); + memset(msp, 0, sizeof(*msp)); + msp->type = MULTITERMINAL; + msp->nsubsym = 1; + msp->subsym = (struct symbol **) calloc(1,sizeof(struct symbol*)); + msp->subsym[0] = origsp; + msp->name = origsp->name; + psp->rhs[psp->nrhs-1] = msp; + } + msp->nsubsym++; + msp->subsym = (struct symbol **) realloc(msp->subsym, + sizeof(struct symbol*)*msp->nsubsym); + msp->subsym[msp->nsubsym-1] = Symbol_new(&x[1]); + if( ISLOWER(x[1]) || ISLOWER(msp->subsym[0]->name[0]) ){ + ErrorMsg(psp->filename,psp->tokenlineno, + "Cannot form a compound containing a non-terminal"); + psp->errorcnt++; + } + }else if( x[0]=='(' && psp->nrhs>0 ){ + psp->state = RHS_ALIAS_1; + }else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Illegal character on RHS of rule: \"%s\".",x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_RULE_ERROR; + } + break; + case RHS_ALIAS_1: + if( ISALPHA(x[0]) ){ + psp->alias[psp->nrhs-1] = x; + psp->state = RHS_ALIAS_2; + }else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "\"%s\" is not a valid alias for the RHS symbol \"%s\"\n", + x,psp->rhs[psp->nrhs-1]->name); + psp->errorcnt++; + psp->state = RESYNC_AFTER_RULE_ERROR; + } + break; + case RHS_ALIAS_2: + if( x[0]==')' ){ + psp->state = IN_RHS; + }else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Missing \")\" following LHS alias name \"%s\".",psp->lhsalias); + psp->errorcnt++; + psp->state = RESYNC_AFTER_RULE_ERROR; + } + break; + case WAITING_FOR_DECL_KEYWORD: + if( ISALPHA(x[0]) ){ + psp->declkeyword = x; + psp->declargslot = 0; + psp->decllinenoslot = 0; + psp->insertLineMacro = 1; + psp->state = WAITING_FOR_DECL_ARG; + if( strcmp(x,"name")==0 ){ + psp->declargslot = &(psp->gp->name); + psp->insertLineMacro = 0; + }else if( strcmp(x,"include")==0 ){ + psp->declargslot = &(psp->gp->include); + }else if( strcmp(x,"code")==0 ){ + psp->declargslot = &(psp->gp->extracode); + }else if( strcmp(x,"token_destructor")==0 ){ + psp->declargslot = &psp->gp->tokendest; + }else if( strcmp(x,"default_destructor")==0 ){ + psp->declargslot = &psp->gp->vardest; + }else if( strcmp(x,"token_prefix")==0 ){ + psp->declargslot = &psp->gp->tokenprefix; + psp->insertLineMacro = 0; + }else if( strcmp(x,"syntax_error")==0 ){ + psp->declargslot = &(psp->gp->error); + }else if( strcmp(x,"parse_accept")==0 ){ + psp->declargslot = &(psp->gp->accept); + }else if( strcmp(x,"parse_failure")==0 ){ + psp->declargslot = &(psp->gp->failure); + }else if( strcmp(x,"stack_overflow")==0 ){ + psp->declargslot = &(psp->gp->overflow); + }else if( strcmp(x,"extra_argument")==0 ){ + psp->declargslot = &(psp->gp->arg); + psp->insertLineMacro = 0; + }else if( strcmp(x,"extra_context")==0 ){ + psp->declargslot = &(psp->gp->ctx); + psp->insertLineMacro = 0; + }else if( strcmp(x,"token_type")==0 ){ + psp->declargslot = &(psp->gp->tokentype); + psp->insertLineMacro = 0; + }else if( strcmp(x,"default_type")==0 ){ + psp->declargslot = &(psp->gp->vartype); + psp->insertLineMacro = 0; + }else if( strcmp(x,"stack_size")==0 ){ + psp->declargslot = &(psp->gp->stacksize); + psp->insertLineMacro = 0; + }else if( strcmp(x,"start_symbol")==0 ){ + psp->declargslot = &(psp->gp->start); + psp->insertLineMacro = 0; + }else if( strcmp(x,"left")==0 ){ + psp->preccounter++; + psp->declassoc = LEFT; + psp->state = WAITING_FOR_PRECEDENCE_SYMBOL; + }else if( strcmp(x,"right")==0 ){ + psp->preccounter++; + psp->declassoc = RIGHT; + psp->state = WAITING_FOR_PRECEDENCE_SYMBOL; + }else if( strcmp(x,"nonassoc")==0 ){ + psp->preccounter++; + psp->declassoc = NONE; + psp->state = WAITING_FOR_PRECEDENCE_SYMBOL; + }else if( strcmp(x,"destructor")==0 ){ + psp->state = WAITING_FOR_DESTRUCTOR_SYMBOL; + }else if( strcmp(x,"type")==0 ){ + psp->state = WAITING_FOR_DATATYPE_SYMBOL; + }else if( strcmp(x,"fallback")==0 ){ + psp->fallback = 0; + psp->state = WAITING_FOR_FALLBACK_ID; + }else if( strcmp(x,"token")==0 ){ + psp->state = WAITING_FOR_TOKEN_NAME; + }else if( strcmp(x,"wildcard")==0 ){ + psp->state = WAITING_FOR_WILDCARD_ID; + }else if( strcmp(x,"token_class")==0 ){ + psp->state = WAITING_FOR_CLASS_ID; + }else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Unknown declaration keyword: \"%%%s\".",x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + } + }else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Illegal declaration keyword: \"%s\".",x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + } + break; + case WAITING_FOR_DESTRUCTOR_SYMBOL: + if( !ISALPHA(x[0]) ){ + ErrorMsg(psp->filename,psp->tokenlineno, + "Symbol name missing after %%destructor keyword"); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + }else{ + struct symbol *sp = Symbol_new(x); + psp->declargslot = &sp->destructor; + psp->decllinenoslot = &sp->destLineno; + psp->insertLineMacro = 1; + psp->state = WAITING_FOR_DECL_ARG; + } + break; + case WAITING_FOR_DATATYPE_SYMBOL: + if( !ISALPHA(x[0]) ){ + ErrorMsg(psp->filename,psp->tokenlineno, + "Symbol name missing after %%type keyword"); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + }else{ + struct symbol *sp = Symbol_find(x); + if((sp) && (sp->datatype)){ + ErrorMsg(psp->filename,psp->tokenlineno, + "Symbol %%type \"%s\" already defined", x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + }else{ + if (!sp){ + sp = Symbol_new(x); + } + psp->declargslot = &sp->datatype; + psp->insertLineMacro = 0; + psp->state = WAITING_FOR_DECL_ARG; + } + } + break; + case WAITING_FOR_PRECEDENCE_SYMBOL: + if( x[0]=='.' ){ + psp->state = WAITING_FOR_DECL_OR_RULE; + }else if( ISUPPER(x[0]) ){ + struct symbol *sp; + sp = Symbol_new(x); + if( sp->prec>=0 ){ + ErrorMsg(psp->filename,psp->tokenlineno, + "Symbol \"%s\" has already be given a precedence.",x); + psp->errorcnt++; + }else{ + sp->prec = psp->preccounter; + sp->assoc = psp->declassoc; + } + }else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Can't assign a precedence to \"%s\".",x); + psp->errorcnt++; + } + break; + case WAITING_FOR_DECL_ARG: + if( x[0]=='{' || x[0]=='\"' || ISALNUM(x[0]) ){ + const char *zOld, *zNew; + char *zBuf, *z; + int nOld, n, nLine = 0, nNew, nBack; + int addLineMacro; + char zLine[50]; + zNew = x; + if( zNew[0]=='"' || zNew[0]=='{' ) zNew++; + nNew = lemonStrlen(zNew); + if( *psp->declargslot ){ + zOld = *psp->declargslot; + }else{ + zOld = ""; + } + nOld = lemonStrlen(zOld); + n = nOld + nNew + 20; + addLineMacro = !psp->gp->nolinenosflag + && psp->insertLineMacro + && psp->tokenlineno>1 + && (psp->decllinenoslot==0 || psp->decllinenoslot[0]!=0); + if( addLineMacro ){ + for(z=psp->filename, nBack=0; *z; z++){ + if( *z=='\\' ) nBack++; + } + lemon_sprintf(zLine, "#line %d ", psp->tokenlineno); + nLine = lemonStrlen(zLine); + n += nLine + lemonStrlen(psp->filename) + nBack; + } + *psp->declargslot = (char *) realloc(*psp->declargslot, n); + zBuf = *psp->declargslot + nOld; + if( addLineMacro ){ + if( nOld && zBuf[-1]!='\n' ){ + *(zBuf++) = '\n'; + } + memcpy(zBuf, zLine, nLine); + zBuf += nLine; + *(zBuf++) = '"'; + for(z=psp->filename; *z; z++){ + if( *z=='\\' ){ + *(zBuf++) = '\\'; + } + *(zBuf++) = *z; + } + *(zBuf++) = '"'; + *(zBuf++) = '\n'; + } + if( psp->decllinenoslot && psp->decllinenoslot[0]==0 ){ + psp->decllinenoslot[0] = psp->tokenlineno; + } + memcpy(zBuf, zNew, nNew); + zBuf += nNew; + *zBuf = 0; + psp->state = WAITING_FOR_DECL_OR_RULE; + }else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Illegal argument to %%%s: %s",psp->declkeyword,x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + } + break; + case WAITING_FOR_FALLBACK_ID: + if( x[0]=='.' ){ + psp->state = WAITING_FOR_DECL_OR_RULE; + }else if( !ISUPPER(x[0]) ){ + ErrorMsg(psp->filename, psp->tokenlineno, + "%%fallback argument \"%s\" should be a token", x); + psp->errorcnt++; + }else{ + struct symbol *sp = Symbol_new(x); + if( psp->fallback==0 ){ + psp->fallback = sp; + }else if( sp->fallback ){ + ErrorMsg(psp->filename, psp->tokenlineno, + "More than one fallback assigned to token %s", x); + psp->errorcnt++; + }else{ + sp->fallback = psp->fallback; + psp->gp->has_fallback = 1; + } + } + break; + case WAITING_FOR_TOKEN_NAME: + /* Tokens do not have to be declared before use. But they can be + ** in order to control their assigned integer number. The number for + ** each token is assigned when it is first seen. So by including + ** + ** %token ONE TWO THREE. + ** + ** early in the grammar file, that assigns small consecutive values + ** to each of the tokens ONE TWO and THREE. + */ + if( x[0]=='.' ){ + psp->state = WAITING_FOR_DECL_OR_RULE; + }else if( !ISUPPER(x[0]) ){ + ErrorMsg(psp->filename, psp->tokenlineno, + "%%token argument \"%s\" should be a token", x); + psp->errorcnt++; + }else{ + (void)Symbol_new(x); + } + break; + case WAITING_FOR_WILDCARD_ID: + if( x[0]=='.' ){ + psp->state = WAITING_FOR_DECL_OR_RULE; + }else if( !ISUPPER(x[0]) ){ + ErrorMsg(psp->filename, psp->tokenlineno, + "%%wildcard argument \"%s\" should be a token", x); + psp->errorcnt++; + }else{ + struct symbol *sp = Symbol_new(x); + if( psp->gp->wildcard==0 ){ + psp->gp->wildcard = sp; + }else{ + ErrorMsg(psp->filename, psp->tokenlineno, + "Extra wildcard to token: %s", x); + psp->errorcnt++; + } + } + break; + case WAITING_FOR_CLASS_ID: + if( !ISLOWER(x[0]) ){ + ErrorMsg(psp->filename, psp->tokenlineno, + "%%token_class must be followed by an identifier: %s", x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + }else if( Symbol_find(x) ){ + ErrorMsg(psp->filename, psp->tokenlineno, + "Symbol \"%s\" already used", x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + }else{ + psp->tkclass = Symbol_new(x); + psp->tkclass->type = MULTITERMINAL; + psp->state = WAITING_FOR_CLASS_TOKEN; + } + break; + case WAITING_FOR_CLASS_TOKEN: + if( x[0]=='.' ){ + psp->state = WAITING_FOR_DECL_OR_RULE; + }else if( ISUPPER(x[0]) || ((x[0]=='|' || x[0]=='/') && ISUPPER(x[1])) ){ + struct symbol *msp = psp->tkclass; + msp->nsubsym++; + msp->subsym = (struct symbol **) realloc(msp->subsym, + sizeof(struct symbol*)*msp->nsubsym); + if( !ISUPPER(x[0]) ) x++; + msp->subsym[msp->nsubsym-1] = Symbol_new(x); + }else{ + ErrorMsg(psp->filename, psp->tokenlineno, + "%%token_class argument \"%s\" should be a token", x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + } + break; + case RESYNC_AFTER_RULE_ERROR: +/* if( x[0]=='.' ) psp->state = WAITING_FOR_DECL_OR_RULE; +** break; */ + case RESYNC_AFTER_DECL_ERROR: + if( x[0]=='.' ) psp->state = WAITING_FOR_DECL_OR_RULE; + if( x[0]=='%' ) psp->state = WAITING_FOR_DECL_KEYWORD; + break; + } +} + +/* The text in the input is part of the argument to an %ifdef or %ifndef. +** Evaluate the text as a boolean expression. Return true or false. +*/ +static int eval_preprocessor_boolean(char *z, int lineno){ + int neg = 0; + int res = 0; + int okTerm = 1; + int i; + for(i=0; z[i]!=0; i++){ + if( ISSPACE(z[i]) ) continue; + if( z[i]=='!' ){ + if( !okTerm ) goto pp_syntax_error; + neg = !neg; + continue; + } + if( z[i]=='|' && z[i+1]=='|' ){ + if( okTerm ) goto pp_syntax_error; + if( res ) return 1; + i++; + okTerm = 1; + continue; + } + if( z[i]=='&' && z[i+1]=='&' ){ + if( okTerm ) goto pp_syntax_error; + if( !res ) return 0; + i++; + okTerm = 1; + continue; + } + if( z[i]=='(' ){ + int k; + int n = 1; + if( !okTerm ) goto pp_syntax_error; + for(k=i+1; z[k]; k++){ + if( z[k]==')' ){ + n--; + if( n==0 ){ + z[k] = 0; + res = eval_preprocessor_boolean(&z[i+1], -1); + z[k] = ')'; + if( res<0 ){ + i = i-res; + goto pp_syntax_error; + } + i = k; + break; + } + }else if( z[k]=='(' ){ + n++; + }else if( z[k]==0 ){ + i = k; + goto pp_syntax_error; + } + } + if( neg ){ + res = !res; + neg = 0; + } + okTerm = 0; + continue; + } + if( ISALPHA(z[i]) ){ + int j, k, n; + if( !okTerm ) goto pp_syntax_error; + for(k=i+1; ISALNUM(z[k]) || z[k]=='_'; k++){} + n = k - i; + res = 0; + for(j=0; j<nDefine; j++){ + if( strncmp(azDefine[j],&z[i],n)==0 && azDefine[j][n]==0 ){ + if( !bDefineUsed[j] ){ + bDefineUsed[j] = 1; + nDefineUsed++; + } + res = 1; + break; + } + } + i = k-1; + if( neg ){ + res = !res; + neg = 0; + } + okTerm = 0; + continue; + } + goto pp_syntax_error; + } + return res; + +pp_syntax_error: + if( lineno>0 ){ + fprintf(stderr, "%%if syntax error on line %d.\n", lineno); + fprintf(stderr, " %.*s <-- syntax error here\n", i+1, z); + exit(1); + }else{ + return -(i+1); + } +} + +/* Run the preprocessor over the input file text. The global variables +** azDefine[0] through azDefine[nDefine-1] contains the names of all defined +** macros. This routine looks for "%ifdef" and "%ifndef" and "%endif" and +** comments them out. Text in between is also commented out as appropriate. +*/ +static void preprocess_input(char *z){ + int i, j, k; + int exclude = 0; + int start = 0; + int lineno = 1; + int start_lineno = 1; + for(i=0; z[i]; i++){ + if( z[i]=='\n' ) lineno++; + if( z[i]!='%' || (i>0 && z[i-1]!='\n') ) continue; + if( strncmp(&z[i],"%endif",6)==0 && ISSPACE(z[i+6]) ){ + if( exclude ){ + exclude--; + if( exclude==0 ){ + for(j=start; j<i; j++) if( z[j]!='\n' ) z[j] = ' '; + } + } + for(j=i; z[j] && z[j]!='\n'; j++) z[j] = ' '; + }else if( strncmp(&z[i],"%else",5)==0 && ISSPACE(z[i+5]) ){ + if( exclude==1){ + exclude = 0; + for(j=start; j<i; j++) if( z[j]!='\n' ) z[j] = ' '; + }else if( exclude==0 ){ + exclude = 1; + start = i; + start_lineno = lineno; + } + for(j=i; z[j] && z[j]!='\n'; j++) z[j] = ' '; + }else if( strncmp(&z[i],"%ifdef ",7)==0 + || strncmp(&z[i],"%if ",4)==0 + || strncmp(&z[i],"%ifndef ",8)==0 ){ + if( exclude ){ + exclude++; + }else{ + int isNot; + int iBool; + for(j=i; z[j] && !ISSPACE(z[j]); j++){} + iBool = j; + isNot = (j==i+7); + while( z[j] && z[j]!='\n' ){ j++; } + k = z[j]; + z[j] = 0; + exclude = eval_preprocessor_boolean(&z[iBool], lineno); + z[j] = k; + if( !isNot ) exclude = !exclude; + if( exclude ){ + start = i; + start_lineno = lineno; + } + } + for(j=i; z[j] && z[j]!='\n'; j++) z[j] = ' '; + } + } + if( exclude ){ + fprintf(stderr,"unterminated %%ifdef starting on line %d\n", start_lineno); + exit(1); + } +} + +/* In spite of its name, this function is really a scanner. It read +** in the entire input file (all at once) then tokenizes it. Each +** token is passed to the function "parseonetoken" which builds all +** the appropriate data structures in the global state vector "gp". +*/ +void Parse(struct lemon *gp) +{ + struct pstate ps; + FILE *fp; + char *filebuf; + unsigned int filesize; + int lineno; + int c; + char *cp, *nextcp; + int startline = 0; + + memset(&ps, '\0', sizeof(ps)); + ps.gp = gp; + ps.filename = gp->filename; + ps.errorcnt = 0; + ps.state = INITIALIZE; + + /* Begin by reading the input file */ + fp = fopen(ps.filename,"rb"); + if( fp==0 ){ + ErrorMsg(ps.filename,0,"Can't open this file for reading."); + gp->errorcnt++; + return; + } + fseek(fp,0,2); + filesize = ftell(fp); + rewind(fp); + filebuf = (char *)malloc( filesize+1 ); + if( filesize>100000000 || filebuf==0 ){ + ErrorMsg(ps.filename,0,"Input file too large."); + free(filebuf); + gp->errorcnt++; + fclose(fp); + return; + } + if( fread(filebuf,1,filesize,fp)!=filesize ){ + ErrorMsg(ps.filename,0,"Can't read in all %d bytes of this file.", + filesize); + free(filebuf); + gp->errorcnt++; + fclose(fp); + return; + } + fclose(fp); + filebuf[filesize] = 0; + + /* Make an initial pass through the file to handle %ifdef and %ifndef */ + preprocess_input(filebuf); + if( gp->printPreprocessed ){ + printf("%s\n", filebuf); + return; + } + + /* Now scan the text of the input file */ + lineno = 1; + for(cp=filebuf; (c= *cp)!=0; ){ + if( c=='\n' ) lineno++; /* Keep track of the line number */ + if( ISSPACE(c) ){ cp++; continue; } /* Skip all white space */ + if( c=='/' && cp[1]=='/' ){ /* Skip C++ style comments */ + cp+=2; + while( (c= *cp)!=0 && c!='\n' ) cp++; + continue; + } + if( c=='/' && cp[1]=='*' ){ /* Skip C style comments */ + cp+=2; + if( (*cp)=='/' ) cp++; + while( (c= *cp)!=0 && (c!='/' || cp[-1]!='*') ){ + if( c=='\n' ) lineno++; + cp++; + } + if( c ) cp++; + continue; + } + ps.tokenstart = cp; /* Mark the beginning of the token */ + ps.tokenlineno = lineno; /* Linenumber on which token begins */ + if( c=='\"' ){ /* String literals */ + cp++; + while( (c= *cp)!=0 && c!='\"' ){ + if( c=='\n' ) lineno++; + cp++; + } + if( c==0 ){ + ErrorMsg(ps.filename,startline, + "String starting on this line is not terminated before " + "the end of the file."); + ps.errorcnt++; + nextcp = cp; + }else{ + nextcp = cp+1; + } + }else if( c=='{' ){ /* A block of C code */ + int level; + cp++; + for(level=1; (c= *cp)!=0 && (level>1 || c!='}'); cp++){ + if( c=='\n' ) lineno++; + else if( c=='{' ) level++; + else if( c=='}' ) level--; + else if( c=='/' && cp[1]=='*' ){ /* Skip comments */ + int prevc; + cp = &cp[2]; + prevc = 0; + while( (c= *cp)!=0 && (c!='/' || prevc!='*') ){ + if( c=='\n' ) lineno++; + prevc = c; + cp++; + } + }else if( c=='/' && cp[1]=='/' ){ /* Skip C++ style comments too */ + cp = &cp[2]; + while( (c= *cp)!=0 && c!='\n' ) cp++; + if( c ) lineno++; + }else if( c=='\'' || c=='\"' ){ /* String a character literals */ + int startchar, prevc; + startchar = c; + prevc = 0; + for(cp++; (c= *cp)!=0 && (c!=startchar || prevc=='\\'); cp++){ + if( c=='\n' ) lineno++; + if( prevc=='\\' ) prevc = 0; + else prevc = c; + } + } + } + if( c==0 ){ + ErrorMsg(ps.filename,ps.tokenlineno, + "C code starting on this line is not terminated before " + "the end of the file."); + ps.errorcnt++; + nextcp = cp; + }else{ + nextcp = cp+1; + } + }else if( ISALNUM(c) ){ /* Identifiers */ + while( (c= *cp)!=0 && (ISALNUM(c) || c=='_') ) cp++; + nextcp = cp; + }else if( c==':' && cp[1]==':' && cp[2]=='=' ){ /* The operator "::=" */ + cp += 3; + nextcp = cp; + }else if( (c=='/' || c=='|') && ISALPHA(cp[1]) ){ + cp += 2; + while( (c = *cp)!=0 && (ISALNUM(c) || c=='_') ) cp++; + nextcp = cp; + }else{ /* All other (one character) operators */ + cp++; + nextcp = cp; + } + c = *cp; + *cp = 0; /* Null terminate the token */ + parseonetoken(&ps); /* Parse the token */ + *cp = (char)c; /* Restore the buffer */ + cp = nextcp; + } + free(filebuf); /* Release the buffer after parsing */ + gp->rule = ps.firstrule; + gp->errorcnt = ps.errorcnt; +} +/*************************** From the file "plink.c" *********************/ +/* +** Routines processing configuration follow-set propagation links +** in the LEMON parser generator. +*/ +static struct plink *plink_freelist = 0; + +/* Allocate a new plink */ +struct plink *Plink_new(void){ + struct plink *newlink; + + if( plink_freelist==0 ){ + int i; + int amt = 100; + plink_freelist = (struct plink *)calloc( amt, sizeof(struct plink) ); + if( plink_freelist==0 ){ + fprintf(stderr, + "Unable to allocate memory for a new follow-set propagation link.\n"); + exit(1); + } + for(i=0; i<amt-1; i++) plink_freelist[i].next = &plink_freelist[i+1]; + plink_freelist[amt-1].next = 0; + } + newlink = plink_freelist; + plink_freelist = plink_freelist->next; + return newlink; +} + +/* Add a plink to a plink list */ +void Plink_add(struct plink **plpp, struct config *cfp) +{ + struct plink *newlink; + newlink = Plink_new(); + newlink->next = *plpp; + *plpp = newlink; + newlink->cfp = cfp; +} + +/* Transfer every plink on the list "from" to the list "to" */ +void Plink_copy(struct plink **to, struct plink *from) +{ + struct plink *nextpl; + while( from ){ + nextpl = from->next; + from->next = *to; + *to = from; + from = nextpl; + } +} + +/* Delete every plink on the list */ +void Plink_delete(struct plink *plp) +{ + struct plink *nextpl; + + while( plp ){ + nextpl = plp->next; + plp->next = plink_freelist; + plink_freelist = plp; + plp = nextpl; + } +} +/*********************** From the file "report.c" **************************/ +/* +** Procedures for generating reports and tables in the LEMON parser generator. +*/ + +/* Generate a filename with the given suffix. Space to hold the +** name comes from malloc() and must be freed by the calling +** function. +*/ +PRIVATE char *file_makename(struct lemon *lemp, const char *suffix) +{ + char *name; + char *cp; + char *filename = lemp->filename; + int sz; + + if( outputDir ){ + cp = strrchr(filename, '/'); + if( cp ) filename = cp + 1; + } + sz = lemonStrlen(filename); + sz += lemonStrlen(suffix); + if( outputDir ) sz += lemonStrlen(outputDir) + 1; + sz += 5; + name = (char*)malloc( sz ); + if( name==0 ){ + fprintf(stderr,"Can't allocate space for a filename.\n"); + exit(1); + } + name[0] = 0; + if( outputDir ){ + lemon_strcpy(name, outputDir); + lemon_strcat(name, "/"); + } + lemon_strcat(name,filename); + cp = strrchr(name,'.'); + if( cp ) *cp = 0; + lemon_strcat(name,suffix); + return name; +} + +/* Open a file with a name based on the name of the input file, +** but with a different (specified) suffix, and return a pointer +** to the stream */ +PRIVATE FILE *file_open( + struct lemon *lemp, + const char *suffix, + const char *mode +){ + FILE *fp; + + if( lemp->outname ) free(lemp->outname); + lemp->outname = file_makename(lemp, suffix); + fp = fopen(lemp->outname,mode); + if( fp==0 && *mode=='w' ){ + fprintf(stderr,"Can't open file \"%s\".\n",lemp->outname); + lemp->errorcnt++; + return 0; + } + return fp; +} + +/* Print the text of a rule +*/ +void rule_print(FILE *out, struct rule *rp){ + int i, j; + fprintf(out, "%s",rp->lhs->name); + /* if( rp->lhsalias ) fprintf(out,"(%s)",rp->lhsalias); */ + fprintf(out," ::="); + for(i=0; i<rp->nrhs; i++){ + struct symbol *sp = rp->rhs[i]; + if( sp->type==MULTITERMINAL ){ + fprintf(out," %s", sp->subsym[0]->name); + for(j=1; j<sp->nsubsym; j++){ + fprintf(out,"|%s", sp->subsym[j]->name); + } + }else{ + fprintf(out," %s", sp->name); + } + /* if( rp->rhsalias[i] ) fprintf(out,"(%s)",rp->rhsalias[i]); */ + } +} + +/* Duplicate the input file without comments and without actions +** on rules */ +void Reprint(struct lemon *lemp) +{ + struct rule *rp; + struct symbol *sp; + int i, j, maxlen, len, ncolumns, skip; + printf("// Reprint of input file \"%s\".\n// Symbols:\n",lemp->filename); + maxlen = 10; + for(i=0; i<lemp->nsymbol; i++){ + sp = lemp->symbols[i]; + len = lemonStrlen(sp->name); + if( len>maxlen ) maxlen = len; + } + ncolumns = 76/(maxlen+5); + if( ncolumns<1 ) ncolumns = 1; + skip = (lemp->nsymbol + ncolumns - 1)/ncolumns; + for(i=0; i<skip; i++){ + printf("//"); + for(j=i; j<lemp->nsymbol; j+=skip){ + sp = lemp->symbols[j]; + assert( sp->index==j ); + printf(" %3d %-*.*s",j,maxlen,maxlen,sp->name); + } + printf("\n"); + } + for(rp=lemp->rule; rp; rp=rp->next){ + rule_print(stdout, rp); + printf("."); + if( rp->precsym ) printf(" [%s]",rp->precsym->name); + /* if( rp->code ) printf("\n %s",rp->code); */ + printf("\n"); + } +} + +/* Print a single rule. +*/ +void RulePrint(FILE *fp, struct rule *rp, int iCursor){ + struct symbol *sp; + int i, j; + fprintf(fp,"%s ::=",rp->lhs->name); + for(i=0; i<=rp->nrhs; i++){ + if( i==iCursor ) fprintf(fp," *"); + if( i==rp->nrhs ) break; + sp = rp->rhs[i]; + if( sp->type==MULTITERMINAL ){ + fprintf(fp," %s", sp->subsym[0]->name); + for(j=1; j<sp->nsubsym; j++){ + fprintf(fp,"|%s",sp->subsym[j]->name); + } + }else{ + fprintf(fp," %s", sp->name); + } + } +} + +/* Print the rule for a configuration. +*/ +void ConfigPrint(FILE *fp, struct config *cfp){ + RulePrint(fp, cfp->rp, cfp->dot); +} + +/* #define TEST */ +#if 0 +/* Print a set */ +PRIVATE void SetPrint(out,set,lemp) +FILE *out; +char *set; +struct lemon *lemp; +{ + int i; + char *spacer; + spacer = ""; + fprintf(out,"%12s[",""); + for(i=0; i<lemp->nterminal; i++){ + if( SetFind(set,i) ){ + fprintf(out,"%s%s",spacer,lemp->symbols[i]->name); + spacer = " "; + } + } + fprintf(out,"]\n"); +} + +/* Print a plink chain */ +PRIVATE void PlinkPrint(out,plp,tag) +FILE *out; +struct plink *plp; +char *tag; +{ + while( plp ){ + fprintf(out,"%12s%s (state %2d) ","",tag,plp->cfp->stp->statenum); + ConfigPrint(out,plp->cfp); + fprintf(out,"\n"); + plp = plp->next; + } +} +#endif + +/* Print an action to the given file descriptor. Return FALSE if +** nothing was actually printed. +*/ +int PrintAction( + struct action *ap, /* The action to print */ + FILE *fp, /* Print the action here */ + int indent /* Indent by this amount */ +){ + int result = 1; + switch( ap->type ){ + case SHIFT: { + struct state *stp = ap->x.stp; + fprintf(fp,"%*s shift %-7d",indent,ap->sp->name,stp->statenum); + break; + } + case REDUCE: { + struct rule *rp = ap->x.rp; + fprintf(fp,"%*s reduce %-7d",indent,ap->sp->name,rp->iRule); + RulePrint(fp, rp, -1); + break; + } + case SHIFTREDUCE: { + struct rule *rp = ap->x.rp; + fprintf(fp,"%*s shift-reduce %-7d",indent,ap->sp->name,rp->iRule); + RulePrint(fp, rp, -1); + break; + } + case ACCEPT: + fprintf(fp,"%*s accept",indent,ap->sp->name); + break; + case ERROR: + fprintf(fp,"%*s error",indent,ap->sp->name); + break; + case SRCONFLICT: + case RRCONFLICT: + fprintf(fp,"%*s reduce %-7d ** Parsing conflict **", + indent,ap->sp->name,ap->x.rp->iRule); + break; + case SSCONFLICT: + fprintf(fp,"%*s shift %-7d ** Parsing conflict **", + indent,ap->sp->name,ap->x.stp->statenum); + break; + case SH_RESOLVED: + if( showPrecedenceConflict ){ + fprintf(fp,"%*s shift %-7d -- dropped by precedence", + indent,ap->sp->name,ap->x.stp->statenum); + }else{ + result = 0; + } + break; + case RD_RESOLVED: + if( showPrecedenceConflict ){ + fprintf(fp,"%*s reduce %-7d -- dropped by precedence", + indent,ap->sp->name,ap->x.rp->iRule); + }else{ + result = 0; + } + break; + case NOT_USED: + result = 0; + break; + } + if( result && ap->spOpt ){ + fprintf(fp," /* because %s==%s */", ap->sp->name, ap->spOpt->name); + } + return result; +} + +/* Generate the "*.out" log file */ +void ReportOutput(struct lemon *lemp) +{ + int i, n; + struct state *stp; + struct config *cfp; + struct action *ap; + struct rule *rp; + FILE *fp; + + fp = file_open(lemp,".out","wb"); + if( fp==0 ) return; + for(i=0; i<lemp->nxstate; i++){ + stp = lemp->sorted[i]; + fprintf(fp,"State %d:\n",stp->statenum); + if( lemp->basisflag ) cfp=stp->bp; + else cfp=stp->cfp; + while( cfp ){ + char buf[20]; + if( cfp->dot==cfp->rp->nrhs ){ + lemon_sprintf(buf,"(%d)",cfp->rp->iRule); + fprintf(fp," %5s ",buf); + }else{ + fprintf(fp," "); + } + ConfigPrint(fp,cfp); + fprintf(fp,"\n"); +#if 0 + SetPrint(fp,cfp->fws,lemp); + PlinkPrint(fp,cfp->fplp,"To "); + PlinkPrint(fp,cfp->bplp,"From"); +#endif + if( lemp->basisflag ) cfp=cfp->bp; + else cfp=cfp->next; + } + fprintf(fp,"\n"); + for(ap=stp->ap; ap; ap=ap->next){ + if( PrintAction(ap,fp,30) ) fprintf(fp,"\n"); + } + fprintf(fp,"\n"); + } + fprintf(fp, "----------------------------------------------------\n"); + fprintf(fp, "Symbols:\n"); + fprintf(fp, "The first-set of non-terminals is shown after the name.\n\n"); + for(i=0; i<lemp->nsymbol; i++){ + int j; + struct symbol *sp; + + sp = lemp->symbols[i]; + fprintf(fp, " %3d: %s", i, sp->name); + if( sp->type==NONTERMINAL ){ + fprintf(fp, ":"); + if( sp->lambda ){ + fprintf(fp, " <lambda>"); + } + for(j=0; j<lemp->nterminal; j++){ + if( sp->firstset && SetFind(sp->firstset, j) ){ + fprintf(fp, " %s", lemp->symbols[j]->name); + } + } + } + if( sp->prec>=0 ) fprintf(fp," (precedence=%d)", sp->prec); + fprintf(fp, "\n"); + } + fprintf(fp, "----------------------------------------------------\n"); + fprintf(fp, "Syntax-only Symbols:\n"); + fprintf(fp, "The following symbols never carry semantic content.\n\n"); + for(i=n=0; i<lemp->nsymbol; i++){ + int w; + struct symbol *sp = lemp->symbols[i]; + if( sp->bContent ) continue; + w = (int)strlen(sp->name); + if( n>0 && n+w>75 ){ + fprintf(fp,"\n"); + n = 0; + } + if( n>0 ){ + fprintf(fp, " "); + n++; + } + fprintf(fp, "%s", sp->name); + n += w; + } + if( n>0 ) fprintf(fp, "\n"); + fprintf(fp, "----------------------------------------------------\n"); + fprintf(fp, "Rules:\n"); + for(rp=lemp->rule; rp; rp=rp->next){ + fprintf(fp, "%4d: ", rp->iRule); + rule_print(fp, rp); + fprintf(fp,"."); + if( rp->precsym ){ + fprintf(fp," [%s precedence=%d]", + rp->precsym->name, rp->precsym->prec); + } + fprintf(fp,"\n"); + } + fclose(fp); + return; +} + +/* Search for the file "name" which is in the same directory as +** the executable */ +PRIVATE char *pathsearch(char *argv0, char *name, int modemask) +{ + const char *pathlist; + char *pathbufptr = 0; + char *pathbuf = 0; + char *path,*cp; + char c; + +#ifdef __WIN32__ + cp = strrchr(argv0,'\\'); +#else + cp = strrchr(argv0,'/'); +#endif + if( cp ){ + c = *cp; + *cp = 0; + path = (char *)malloc( lemonStrlen(argv0) + lemonStrlen(name) + 2 ); + if( path ) lemon_sprintf(path,"%s/%s",argv0,name); + *cp = c; + }else{ + pathlist = getenv("PATH"); + if( pathlist==0 ) pathlist = ".:/bin:/usr/bin"; + pathbuf = (char *) malloc( lemonStrlen(pathlist) + 1 ); + path = (char *)malloc( lemonStrlen(pathlist)+lemonStrlen(name)+2 ); + if( (pathbuf != 0) && (path!=0) ){ + pathbufptr = pathbuf; + lemon_strcpy(pathbuf, pathlist); + while( *pathbuf ){ + cp = strchr(pathbuf,':'); + if( cp==0 ) cp = &pathbuf[lemonStrlen(pathbuf)]; + c = *cp; + *cp = 0; + lemon_sprintf(path,"%s/%s",pathbuf,name); + *cp = c; + if( c==0 ) pathbuf[0] = 0; + else pathbuf = &cp[1]; + if( access(path,modemask)==0 ) break; + } + } + free(pathbufptr); + } + return path; +} + +/* Given an action, compute the integer value for that action +** which is to be put in the action table of the generated machine. +** Return negative if no action should be generated. +*/ +PRIVATE int compute_action(struct lemon *lemp, struct action *ap) +{ + int act; + switch( ap->type ){ + case SHIFT: act = ap->x.stp->statenum; break; + case SHIFTREDUCE: { + /* Since a SHIFT is inherient after a prior REDUCE, convert any + ** SHIFTREDUCE action with a nonterminal on the LHS into a simple + ** REDUCE action: */ + if( ap->sp->index>=lemp->nterminal + && (lemp->errsym==0 || ap->sp->index!=lemp->errsym->index) + ){ + act = lemp->minReduce + ap->x.rp->iRule; + }else{ + act = lemp->minShiftReduce + ap->x.rp->iRule; + } + break; + } + case REDUCE: act = lemp->minReduce + ap->x.rp->iRule; break; + case ERROR: act = lemp->errAction; break; + case ACCEPT: act = lemp->accAction; break; + default: act = -1; break; + } + return act; +} + +#define LINESIZE 1000 +/* The next cluster of routines are for reading the template file +** and writing the results to the generated parser */ +/* The first function transfers data from "in" to "out" until +** a line is seen which begins with "%%". The line number is +** tracked. +** +** if name!=0, then any word that begin with "Parse" is changed to +** begin with *name instead. +*/ +PRIVATE void tplt_xfer(char *name, FILE *in, FILE *out, int *lineno) +{ + int i, iStart; + char line[LINESIZE]; + while( fgets(line,LINESIZE,in) && (line[0]!='%' || line[1]!='%') ){ + (*lineno)++; + iStart = 0; + if( name ){ + for(i=0; line[i]; i++){ + if( line[i]=='P' && strncmp(&line[i],"Parse",5)==0 + && (i==0 || !ISALPHA(line[i-1])) + ){ + if( i>iStart ) fprintf(out,"%.*s",i-iStart,&line[iStart]); + fprintf(out,"%s",name); + i += 4; + iStart = i+1; + } + } + } + fprintf(out,"%s",&line[iStart]); + } +} + +/* Skip forward past the header of the template file to the first "%%" +*/ +PRIVATE void tplt_skip_header(FILE *in, int *lineno) +{ + char line[LINESIZE]; + while( fgets(line,LINESIZE,in) && (line[0]!='%' || line[1]!='%') ){ + (*lineno)++; + } +} + +/* The next function finds the template file and opens it, returning +** a pointer to the opened file. */ +PRIVATE FILE *tplt_open(struct lemon *lemp) +{ + static char templatename[] = "lempar.c"; + char buf[1000]; + FILE *in; + char *tpltname; + char *toFree = 0; + char *cp; + + /* first, see if user specified a template filename on the command line. */ + if (user_templatename != 0) { + if( access(user_templatename,004)==-1 ){ + fprintf(stderr,"Can't find the parser driver template file \"%s\".\n", + user_templatename); + lemp->errorcnt++; + return 0; + } + in = fopen(user_templatename,"rb"); + if( in==0 ){ + fprintf(stderr,"Can't open the template file \"%s\".\n", + user_templatename); + lemp->errorcnt++; + return 0; + } + return in; + } + + cp = strrchr(lemp->filename,'.'); + if( cp ){ + lemon_sprintf(buf,"%.*s.lt",(int)(cp-lemp->filename),lemp->filename); + }else{ + lemon_sprintf(buf,"%s.lt",lemp->filename); + } + if( access(buf,004)==0 ){ + tpltname = buf; + }else if( access(templatename,004)==0 ){ + tpltname = templatename; + }else{ + toFree = tpltname = pathsearch(lemp->argv[0],templatename,0); + } + if( tpltname==0 ){ + fprintf(stderr,"Can't find the parser driver template file \"%s\".\n", + templatename); + lemp->errorcnt++; + return 0; + } + in = fopen(tpltname,"rb"); + if( in==0 ){ + fprintf(stderr,"Can't open the template file \"%s\".\n",tpltname); + lemp->errorcnt++; + } + free(toFree); + return in; +} + +/* Print a #line directive line to the output file. */ +PRIVATE void tplt_linedir(FILE *out, int lineno, char *filename) +{ + fprintf(out,"#line %d \"",lineno); + while( *filename ){ + if( *filename == '\\' ) putc('\\',out); + putc(*filename,out); + filename++; + } + fprintf(out,"\"\n"); +} + +/* Print a string to the file and keep the linenumber up to date */ +PRIVATE void tplt_print(FILE *out, struct lemon *lemp, char *str, int *lineno) +{ + if( str==0 ) return; + while( *str ){ + putc(*str,out); + if( *str=='\n' ) (*lineno)++; + str++; + } + if( str[-1]!='\n' ){ + putc('\n',out); + (*lineno)++; + } + if (!lemp->nolinenosflag) { + (*lineno)++; tplt_linedir(out,*lineno,lemp->outname); + } + return; +} + +/* +** The following routine emits code for the destructor for the +** symbol sp +*/ +void emit_destructor_code( + FILE *out, + struct symbol *sp, + struct lemon *lemp, + int *lineno +){ + char *cp = 0; + + if( sp->type==TERMINAL ){ + cp = lemp->tokendest; + if( cp==0 ) return; + fprintf(out,"{\n"); (*lineno)++; + }else if( sp->destructor ){ + cp = sp->destructor; + fprintf(out,"{\n"); (*lineno)++; + if( !lemp->nolinenosflag ){ + (*lineno)++; + tplt_linedir(out,sp->destLineno,lemp->filename); + } + }else if( lemp->vardest ){ + cp = lemp->vardest; + if( cp==0 ) return; + fprintf(out,"{\n"); (*lineno)++; + }else{ + assert( 0 ); /* Cannot happen */ + } + for(; *cp; cp++){ + if( *cp=='$' && cp[1]=='$' ){ + fprintf(out,"(yypminor->yy%d)",sp->dtnum); + cp++; + continue; + } + if( *cp=='\n' ) (*lineno)++; + fputc(*cp,out); + } + fprintf(out,"\n"); (*lineno)++; + if (!lemp->nolinenosflag) { + (*lineno)++; tplt_linedir(out,*lineno,lemp->outname); + } + fprintf(out,"}\n"); (*lineno)++; + return; +} + +/* +** Return TRUE (non-zero) if the given symbol has a destructor. +*/ +int has_destructor(struct symbol *sp, struct lemon *lemp) +{ + int ret; + if( sp->type==TERMINAL ){ + ret = lemp->tokendest!=0; + }else{ + ret = lemp->vardest!=0 || sp->destructor!=0; + } + return ret; +} + +/* +** Append text to a dynamically allocated string. If zText is 0 then +** reset the string to be empty again. Always return the complete text +** of the string (which is overwritten with each call). +** +** n bytes of zText are stored. If n==0 then all of zText up to the first +** \000 terminator is stored. zText can contain up to two instances of +** %d. The values of p1 and p2 are written into the first and second +** %d. +** +** If n==-1, then the previous character is overwritten. +*/ +PRIVATE char *append_str(const char *zText, int n, int p1, int p2){ + static char empty[1] = { 0 }; + static char *z = 0; + static int alloced = 0; + static int used = 0; + int c; + char zInt[40]; + if( zText==0 ){ + if( used==0 && z!=0 ) z[0] = 0; + used = 0; + return z; + } + if( n<=0 ){ + if( n<0 ){ + used += n; + assert( used>=0 ); + } + n = lemonStrlen(zText); + } + if( (int) (n+sizeof(zInt)*2+used) >= alloced ){ + alloced = n + sizeof(zInt)*2 + used + 200; + z = (char *) realloc(z, alloced); + } + if( z==0 ) return empty; + while( n-- > 0 ){ + c = *(zText++); + if( c=='%' && n>0 && zText[0]=='d' ){ + lemon_sprintf(zInt, "%d", p1); + p1 = p2; + lemon_strcpy(&z[used], zInt); + used += lemonStrlen(&z[used]); + zText++; + n--; + }else{ + z[used++] = (char)c; + } + } + z[used] = 0; + return z; +} + +/* +** Write and transform the rp->code string so that symbols are expanded. +** Populate the rp->codePrefix and rp->codeSuffix strings, as appropriate. +** +** Return 1 if the expanded code requires that "yylhsminor" local variable +** to be defined. +*/ +PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){ + char *cp, *xp; + int i; + int rc = 0; /* True if yylhsminor is used */ + int dontUseRhs0 = 0; /* If true, use of left-most RHS label is illegal */ + const char *zSkip = 0; /* The zOvwrt comment within rp->code, or NULL */ + char lhsused = 0; /* True if the LHS element has been used */ + char lhsdirect; /* True if LHS writes directly into stack */ + char used[MAXRHS]; /* True for each RHS element which is used */ + char zLhs[50]; /* Convert the LHS symbol into this string */ + char zOvwrt[900]; /* Comment that to allow LHS to overwrite RHS */ + + for(i=0; i<rp->nrhs; i++) used[i] = 0; + lhsused = 0; + + if( rp->code==0 ){ + static char newlinestr[2] = { '\n', '\0' }; + rp->code = newlinestr; + rp->line = rp->ruleline; + rp->noCode = 1; + }else{ + rp->noCode = 0; + } + + + if( rp->nrhs==0 ){ + /* If there are no RHS symbols, then writing directly to the LHS is ok */ + lhsdirect = 1; + }else if( rp->rhsalias[0]==0 ){ + /* The left-most RHS symbol has no value. LHS direct is ok. But + ** we have to call the destructor on the RHS symbol first. */ + lhsdirect = 1; + if( has_destructor(rp->rhs[0],lemp) ){ + append_str(0,0,0,0); + append_str(" yy_destructor(yypParser,%d,&yymsp[%d].minor);\n", 0, + rp->rhs[0]->index,1-rp->nrhs); + rp->codePrefix = Strsafe(append_str(0,0,0,0)); + rp->noCode = 0; + } + }else if( rp->lhsalias==0 ){ + /* There is no LHS value symbol. */ + lhsdirect = 1; + }else if( strcmp(rp->lhsalias,rp->rhsalias[0])==0 ){ + /* The LHS symbol and the left-most RHS symbol are the same, so + ** direct writing is allowed */ + lhsdirect = 1; + lhsused = 1; + used[0] = 1; + if( rp->lhs->dtnum!=rp->rhs[0]->dtnum ){ + ErrorMsg(lemp->filename,rp->ruleline, + "%s(%s) and %s(%s) share the same label but have " + "different datatypes.", + rp->lhs->name, rp->lhsalias, rp->rhs[0]->name, rp->rhsalias[0]); + lemp->errorcnt++; + } + }else{ + lemon_sprintf(zOvwrt, "/*%s-overwrites-%s*/", + rp->lhsalias, rp->rhsalias[0]); + zSkip = strstr(rp->code, zOvwrt); + if( zSkip!=0 ){ + /* The code contains a special comment that indicates that it is safe + ** for the LHS label to overwrite left-most RHS label. */ + lhsdirect = 1; + }else{ + lhsdirect = 0; + } + } + if( lhsdirect ){ + sprintf(zLhs, "yymsp[%d].minor.yy%d",1-rp->nrhs,rp->lhs->dtnum); + }else{ + rc = 1; + sprintf(zLhs, "yylhsminor.yy%d",rp->lhs->dtnum); + } + + append_str(0,0,0,0); + + /* This const cast is wrong but harmless, if we're careful. */ + for(cp=(char *)rp->code; *cp; cp++){ + if( cp==zSkip ){ + append_str(zOvwrt,0,0,0); + cp += lemonStrlen(zOvwrt)-1; + dontUseRhs0 = 1; + continue; + } + if( ISALPHA(*cp) && (cp==rp->code || (!ISALNUM(cp[-1]) && cp[-1]!='_')) ){ + char saved; + for(xp= &cp[1]; ISALNUM(*xp) || *xp=='_'; xp++); + saved = *xp; + *xp = 0; + if( rp->lhsalias && strcmp(cp,rp->lhsalias)==0 ){ + append_str(zLhs,0,0,0); + cp = xp; + lhsused = 1; + }else{ + for(i=0; i<rp->nrhs; i++){ + if( rp->rhsalias[i] && strcmp(cp,rp->rhsalias[i])==0 ){ + if( i==0 && dontUseRhs0 ){ + ErrorMsg(lemp->filename,rp->ruleline, + "Label %s used after '%s'.", + rp->rhsalias[0], zOvwrt); + lemp->errorcnt++; + }else if( cp!=rp->code && cp[-1]=='@' ){ + /* If the argument is of the form @X then substituted + ** the token number of X, not the value of X */ + append_str("yymsp[%d].major",-1,i-rp->nrhs+1,0); + }else{ + struct symbol *sp = rp->rhs[i]; + int dtnum; + if( sp->type==MULTITERMINAL ){ + dtnum = sp->subsym[0]->dtnum; + }else{ + dtnum = sp->dtnum; + } + append_str("yymsp[%d].minor.yy%d",0,i-rp->nrhs+1, dtnum); + } + cp = xp; + used[i] = 1; + break; + } + } + } + *xp = saved; + } + append_str(cp, 1, 0, 0); + } /* End loop */ + + /* Main code generation completed */ + cp = append_str(0,0,0,0); + if( cp && cp[0] ) rp->code = Strsafe(cp); + append_str(0,0,0,0); + + /* Check to make sure the LHS has been used */ + if( rp->lhsalias && !lhsused ){ + ErrorMsg(lemp->filename,rp->ruleline, + "Label \"%s\" for \"%s(%s)\" is never used.", + rp->lhsalias,rp->lhs->name,rp->lhsalias); + lemp->errorcnt++; + } + + /* Generate destructor code for RHS minor values which are not referenced. + ** Generate error messages for unused labels and duplicate labels. + */ + for(i=0; i<rp->nrhs; i++){ + if( rp->rhsalias[i] ){ + if( i>0 ){ + int j; + if( rp->lhsalias && strcmp(rp->lhsalias,rp->rhsalias[i])==0 ){ + ErrorMsg(lemp->filename,rp->ruleline, + "%s(%s) has the same label as the LHS but is not the left-most " + "symbol on the RHS.", + rp->rhs[i]->name, rp->rhsalias[i]); + lemp->errorcnt++; + } + for(j=0; j<i; j++){ + if( rp->rhsalias[j] && strcmp(rp->rhsalias[j],rp->rhsalias[i])==0 ){ + ErrorMsg(lemp->filename,rp->ruleline, + "Label %s used for multiple symbols on the RHS of a rule.", + rp->rhsalias[i]); + lemp->errorcnt++; + break; + } + } + } + if( !used[i] ){ + ErrorMsg(lemp->filename,rp->ruleline, + "Label %s for \"%s(%s)\" is never used.", + rp->rhsalias[i],rp->rhs[i]->name,rp->rhsalias[i]); + lemp->errorcnt++; + } + }else if( i>0 && has_destructor(rp->rhs[i],lemp) ){ + append_str(" yy_destructor(yypParser,%d,&yymsp[%d].minor);\n", 0, + rp->rhs[i]->index,i-rp->nrhs+1); + } + } + + /* If unable to write LHS values directly into the stack, write the + ** saved LHS value now. */ + if( lhsdirect==0 ){ + append_str(" yymsp[%d].minor.yy%d = ", 0, 1-rp->nrhs, rp->lhs->dtnum); + append_str(zLhs, 0, 0, 0); + append_str(";\n", 0, 0, 0); + } + + /* Suffix code generation complete */ + cp = append_str(0,0,0,0); + if( cp && cp[0] ){ + rp->codeSuffix = Strsafe(cp); + rp->noCode = 0; + } + + return rc; +} + +/* +** Generate code which executes when the rule "rp" is reduced. Write +** the code to "out". Make sure lineno stays up-to-date. +*/ +PRIVATE void emit_code( + FILE *out, + struct rule *rp, + struct lemon *lemp, + int *lineno +){ + const char *cp; + + /* Setup code prior to the #line directive */ + if( rp->codePrefix && rp->codePrefix[0] ){ + fprintf(out, "{%s", rp->codePrefix); + for(cp=rp->codePrefix; *cp; cp++){ if( *cp=='\n' ) (*lineno)++; } + } + + /* Generate code to do the reduce action */ + if( rp->code ){ + if( !lemp->nolinenosflag ){ + (*lineno)++; + tplt_linedir(out,rp->line,lemp->filename); + } + fprintf(out,"{%s",rp->code); + for(cp=rp->code; *cp; cp++){ if( *cp=='\n' ) (*lineno)++; } + fprintf(out,"}\n"); (*lineno)++; + if( !lemp->nolinenosflag ){ + (*lineno)++; + tplt_linedir(out,*lineno,lemp->outname); + } + } + + /* Generate breakdown code that occurs after the #line directive */ + if( rp->codeSuffix && rp->codeSuffix[0] ){ + fprintf(out, "%s", rp->codeSuffix); + for(cp=rp->codeSuffix; *cp; cp++){ if( *cp=='\n' ) (*lineno)++; } + } + + if( rp->codePrefix ){ + fprintf(out, "}\n"); (*lineno)++; + } + + return; +} + +/* +** Print the definition of the union used for the parser's data stack. +** This union contains fields for every possible data type for tokens +** and nonterminals. In the process of computing and printing this +** union, also set the ".dtnum" field of every terminal and nonterminal +** symbol. +*/ +void print_stack_union( + FILE *out, /* The output stream */ + struct lemon *lemp, /* The main info structure for this parser */ + int *plineno, /* Pointer to the line number */ + int mhflag /* True if generating makeheaders output */ +){ + int lineno; /* The line number of the output */ + char **types; /* A hash table of datatypes */ + int arraysize; /* Size of the "types" array */ + int maxdtlength; /* Maximum length of any ".datatype" field. */ + char *stddt; /* Standardized name for a datatype */ + int i,j; /* Loop counters */ + unsigned hash; /* For hashing the name of a type */ + const char *name; /* Name of the parser */ + + /* Allocate and initialize types[] and allocate stddt[] */ + arraysize = lemp->nsymbol * 2; + types = (char**)calloc( arraysize, sizeof(char*) ); + if( types==0 ){ + fprintf(stderr,"Out of memory.\n"); + exit(1); + } + for(i=0; i<arraysize; i++) types[i] = 0; + maxdtlength = 0; + if( lemp->vartype ){ + maxdtlength = lemonStrlen(lemp->vartype); + } + for(i=0; i<lemp->nsymbol; i++){ + int len; + struct symbol *sp = lemp->symbols[i]; + if( sp->datatype==0 ) continue; + len = lemonStrlen(sp->datatype); + if( len>maxdtlength ) maxdtlength = len; + } + stddt = (char*)malloc( maxdtlength*2 + 1 ); + if( stddt==0 ){ + fprintf(stderr,"Out of memory.\n"); + exit(1); + } + + /* Build a hash table of datatypes. The ".dtnum" field of each symbol + ** is filled in with the hash index plus 1. A ".dtnum" value of 0 is + ** used for terminal symbols. If there is no %default_type defined then + ** 0 is also used as the .dtnum value for nonterminals which do not specify + ** a datatype using the %type directive. + */ + for(i=0; i<lemp->nsymbol; i++){ + struct symbol *sp = lemp->symbols[i]; + char *cp; + if( sp==lemp->errsym ){ + sp->dtnum = arraysize+1; + continue; + } + if( sp->type!=NONTERMINAL || (sp->datatype==0 && lemp->vartype==0) ){ + sp->dtnum = 0; + continue; + } + cp = sp->datatype; + if( cp==0 ) cp = lemp->vartype; + j = 0; + while( ISSPACE(*cp) ) cp++; + while( *cp ) stddt[j++] = *cp++; + while( j>0 && ISSPACE(stddt[j-1]) ) j--; + stddt[j] = 0; + if( lemp->tokentype && strcmp(stddt, lemp->tokentype)==0 ){ + sp->dtnum = 0; + continue; + } + hash = 0; + for(j=0; stddt[j]; j++){ + hash = hash*53 + stddt[j]; + } + hash = (hash & 0x7fffffff)%arraysize; + while( types[hash] ){ + if( strcmp(types[hash],stddt)==0 ){ + sp->dtnum = hash + 1; + break; + } + hash++; + if( hash>=(unsigned)arraysize ) hash = 0; + } + if( types[hash]==0 ){ + sp->dtnum = hash + 1; + types[hash] = (char*)malloc( lemonStrlen(stddt)+1 ); + if( types[hash]==0 ){ + fprintf(stderr,"Out of memory.\n"); + exit(1); + } + lemon_strcpy(types[hash],stddt); + } + } + + /* Print out the definition of YYTOKENTYPE and YYMINORTYPE */ + name = lemp->name ? lemp->name : "Parse"; + lineno = *plineno; + if( mhflag ){ fprintf(out,"#if INTERFACE\n"); lineno++; } + fprintf(out,"#define %sTOKENTYPE %s\n",name, + lemp->tokentype?lemp->tokentype:"void*"); lineno++; + if( mhflag ){ fprintf(out,"#endif\n"); lineno++; } + fprintf(out,"typedef union {\n"); lineno++; + fprintf(out," int yyinit;\n"); lineno++; + fprintf(out," %sTOKENTYPE yy0;\n",name); lineno++; + for(i=0; i<arraysize; i++){ + if( types[i]==0 ) continue; + fprintf(out," %s yy%d;\n",types[i],i+1); lineno++; + free(types[i]); + } + if( lemp->errsym && lemp->errsym->useCnt ){ + fprintf(out," int yy%d;\n",lemp->errsym->dtnum); lineno++; + } + free(stddt); + free(types); + fprintf(out,"} YYMINORTYPE;\n"); lineno++; + *plineno = lineno; +} + +/* +** Return the name of a C datatype able to represent values between +** lwr and upr, inclusive. If pnByte!=NULL then also write the sizeof +** for that type (1, 2, or 4) into *pnByte. +*/ +static const char *minimum_size_type(int lwr, int upr, int *pnByte){ + const char *zType = "int"; + int nByte = 4; + if( lwr>=0 ){ + if( upr<=255 ){ + zType = "unsigned char"; + nByte = 1; + }else if( upr<65535 ){ + zType = "unsigned short int"; + nByte = 2; + }else{ + zType = "unsigned int"; + nByte = 4; + } + }else if( lwr>=-127 && upr<=127 ){ + zType = "signed char"; + nByte = 1; + }else if( lwr>=-32767 && upr<32767 ){ + zType = "short"; + nByte = 2; + } + if( pnByte ) *pnByte = nByte; + return zType; +} + +/* +** Each state contains a set of token transaction and a set of +** nonterminal transactions. Each of these sets makes an instance +** of the following structure. An array of these structures is used +** to order the creation of entries in the yy_action[] table. +*/ +struct axset { + struct state *stp; /* A pointer to a state */ + int isTkn; /* True to use tokens. False for non-terminals */ + int nAction; /* Number of actions */ + int iOrder; /* Original order of action sets */ +}; + +/* +** Compare to axset structures for sorting purposes +*/ +static int axset_compare(const void *a, const void *b){ + struct axset *p1 = (struct axset*)a; + struct axset *p2 = (struct axset*)b; + int c; + c = p2->nAction - p1->nAction; + if( c==0 ){ + c = p1->iOrder - p2->iOrder; + } + assert( c!=0 || p1==p2 ); + return c; +} + +/* +** Write text on "out" that describes the rule "rp". +*/ +static void writeRuleText(FILE *out, struct rule *rp){ + int j; + fprintf(out,"%s ::=", rp->lhs->name); + for(j=0; j<rp->nrhs; j++){ + struct symbol *sp = rp->rhs[j]; + if( sp->type!=MULTITERMINAL ){ + fprintf(out," %s", sp->name); + }else{ + int k; + fprintf(out," %s", sp->subsym[0]->name); + for(k=1; k<sp->nsubsym; k++){ + fprintf(out,"|%s",sp->subsym[k]->name); + } + } + } +} + + +/* Generate C source code for the parser */ +void ReportTable( + struct lemon *lemp, + int mhflag, /* Output in makeheaders format if true */ + int sqlFlag /* Generate the *.sql file too */ +){ + FILE *out, *in, *sql; + int lineno; + struct state *stp; + struct action *ap; + struct rule *rp; + struct acttab *pActtab; + int i, j, n, sz; + int nLookAhead; + int szActionType; /* sizeof(YYACTIONTYPE) */ + int szCodeType; /* sizeof(YYCODETYPE) */ + const char *name; + int mnTknOfst, mxTknOfst; + int mnNtOfst, mxNtOfst; + struct axset *ax; + char *prefix; + + lemp->minShiftReduce = lemp->nstate; + lemp->errAction = lemp->minShiftReduce + lemp->nrule; + lemp->accAction = lemp->errAction + 1; + lemp->noAction = lemp->accAction + 1; + lemp->minReduce = lemp->noAction + 1; + lemp->maxAction = lemp->minReduce + lemp->nrule; + + in = tplt_open(lemp); + if( in==0 ) return; + out = file_open(lemp,".c","wb"); + if( out==0 ){ + fclose(in); + return; + } + if( sqlFlag==0 ){ + sql = 0; + }else{ + sql = file_open(lemp, ".sql", "wb"); + if( sql==0 ){ + fclose(in); + fclose(out); + return; + } + fprintf(sql, + "BEGIN;\n" + "CREATE TABLE symbol(\n" + " id INTEGER PRIMARY KEY,\n" + " name TEXT NOT NULL,\n" + " isTerminal BOOLEAN NOT NULL,\n" + " fallback INTEGER REFERENCES symbol" + " DEFERRABLE INITIALLY DEFERRED\n" + ");\n" + ); + for(i=0; i<lemp->nsymbol; i++){ + fprintf(sql, + "INSERT INTO symbol(id,name,isTerminal,fallback)" + "VALUES(%d,'%s',%s", + i, lemp->symbols[i]->name, + i<lemp->nterminal ? "TRUE" : "FALSE" + ); + if( lemp->symbols[i]->fallback ){ + fprintf(sql, ",%d);\n", lemp->symbols[i]->fallback->index); + }else{ + fprintf(sql, ",NULL);\n"); + } + } + fprintf(sql, + "CREATE TABLE rule(\n" + " ruleid INTEGER PRIMARY KEY,\n" + " lhs INTEGER REFERENCES symbol(id),\n" + " txt TEXT\n" + ");\n" + "CREATE TABLE rulerhs(\n" + " ruleid INTEGER REFERENCES rule(ruleid),\n" + " pos INTEGER,\n" + " sym INTEGER REFERENCES symbol(id)\n" + ");\n" + ); + for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){ + assert( i==rp->iRule ); + fprintf(sql, + "INSERT INTO rule(ruleid,lhs,txt)VALUES(%d,%d,'", + rp->iRule, rp->lhs->index + ); + writeRuleText(sql, rp); + fprintf(sql,"');\n"); + for(j=0; j<rp->nrhs; j++){ + struct symbol *sp = rp->rhs[j]; + if( sp->type!=MULTITERMINAL ){ + fprintf(sql, + "INSERT INTO rulerhs(ruleid,pos,sym)VALUES(%d,%d,%d);\n", + i,j,sp->index + ); + }else{ + int k; + for(k=0; k<sp->nsubsym; k++){ + fprintf(sql, + "INSERT INTO rulerhs(ruleid,pos,sym)VALUES(%d,%d,%d);\n", + i,j,sp->subsym[k]->index + ); + } + } + } + } + fprintf(sql, "COMMIT;\n"); + } + lineno = 1; + + fprintf(out, + "/* This file is automatically generated by Lemon from input grammar\n" + "** source file \"%s\"", lemp->filename); lineno++; + if( nDefineUsed==0 ){ + fprintf(out, ".\n*/\n"); lineno += 2; + }else{ + fprintf(out, " with these options:\n**\n"); lineno += 2; + for(i=0; i<nDefine; i++){ + if( !bDefineUsed[i] ) continue; + fprintf(out, "** -D%s\n", azDefine[i]); lineno++; + } + fprintf(out, "*/\n"); lineno++; + } + + /* The first %include directive begins with a C-language comment, + ** then skip over the header comment of the template file + */ + if( lemp->include==0 ) lemp->include = ""; + for(i=0; ISSPACE(lemp->include[i]); i++){ + if( lemp->include[i]=='\n' ){ + lemp->include += i+1; + i = -1; + } + } + if( lemp->include[0]=='/' ){ + tplt_skip_header(in,&lineno); + }else{ + tplt_xfer(lemp->name,in,out,&lineno); + } + + /* Generate the include code, if any */ + tplt_print(out,lemp,lemp->include,&lineno); + if( mhflag ){ + char *incName = file_makename(lemp, ".h"); + fprintf(out,"#include \"%s\"\n", incName); lineno++; + free(incName); + } + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate #defines for all tokens */ + if( lemp->tokenprefix ) prefix = lemp->tokenprefix; + else prefix = ""; + if( mhflag ){ + fprintf(out,"#if INTERFACE\n"); lineno++; + }else{ + fprintf(out,"#ifndef %s%s\n", prefix, lemp->symbols[1]->name); + } + for(i=1; i<lemp->nterminal; i++){ + fprintf(out,"#define %s%-30s %2d\n",prefix,lemp->symbols[i]->name,i); + lineno++; + } + fprintf(out,"#endif\n"); lineno++; + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate the defines */ + fprintf(out,"#define YYCODETYPE %s\n", + minimum_size_type(0, lemp->nsymbol, &szCodeType)); lineno++; + fprintf(out,"#define YYNOCODE %d\n",lemp->nsymbol); lineno++; + fprintf(out,"#define YYACTIONTYPE %s\n", + minimum_size_type(0,lemp->maxAction,&szActionType)); lineno++; + if( lemp->wildcard ){ + fprintf(out,"#define YYWILDCARD %d\n", + lemp->wildcard->index); lineno++; + } + print_stack_union(out,lemp,&lineno,mhflag); + fprintf(out, "#ifndef YYSTACKDEPTH\n"); lineno++; + if( lemp->stacksize ){ + fprintf(out,"#define YYSTACKDEPTH %s\n",lemp->stacksize); lineno++; + }else{ + fprintf(out,"#define YYSTACKDEPTH 100\n"); lineno++; + } + fprintf(out, "#endif\n"); lineno++; + if( mhflag ){ + fprintf(out,"#if INTERFACE\n"); lineno++; + } + name = lemp->name ? lemp->name : "Parse"; + if( lemp->arg && lemp->arg[0] ){ + i = lemonStrlen(lemp->arg); + while( i>=1 && ISSPACE(lemp->arg[i-1]) ) i--; + while( i>=1 && (ISALNUM(lemp->arg[i-1]) || lemp->arg[i-1]=='_') ) i--; + fprintf(out,"#define %sARG_SDECL %s;\n",name,lemp->arg); lineno++; + fprintf(out,"#define %sARG_PDECL ,%s\n",name,lemp->arg); lineno++; + fprintf(out,"#define %sARG_PARAM ,%s\n",name,&lemp->arg[i]); lineno++; + fprintf(out,"#define %sARG_FETCH %s=yypParser->%s;\n", + name,lemp->arg,&lemp->arg[i]); lineno++; + fprintf(out,"#define %sARG_STORE yypParser->%s=%s;\n", + name,&lemp->arg[i],&lemp->arg[i]); lineno++; + }else{ + fprintf(out,"#define %sARG_SDECL\n",name); lineno++; + fprintf(out,"#define %sARG_PDECL\n",name); lineno++; + fprintf(out,"#define %sARG_PARAM\n",name); lineno++; + fprintf(out,"#define %sARG_FETCH\n",name); lineno++; + fprintf(out,"#define %sARG_STORE\n",name); lineno++; + } + if( lemp->ctx && lemp->ctx[0] ){ + i = lemonStrlen(lemp->ctx); + while( i>=1 && ISSPACE(lemp->ctx[i-1]) ) i--; + while( i>=1 && (ISALNUM(lemp->ctx[i-1]) || lemp->ctx[i-1]=='_') ) i--; + fprintf(out,"#define %sCTX_SDECL %s;\n",name,lemp->ctx); lineno++; + fprintf(out,"#define %sCTX_PDECL ,%s\n",name,lemp->ctx); lineno++; + fprintf(out,"#define %sCTX_PARAM ,%s\n",name,&lemp->ctx[i]); lineno++; + fprintf(out,"#define %sCTX_FETCH %s=yypParser->%s;\n", + name,lemp->ctx,&lemp->ctx[i]); lineno++; + fprintf(out,"#define %sCTX_STORE yypParser->%s=%s;\n", + name,&lemp->ctx[i],&lemp->ctx[i]); lineno++; + }else{ + fprintf(out,"#define %sCTX_SDECL\n",name); lineno++; + fprintf(out,"#define %sCTX_PDECL\n",name); lineno++; + fprintf(out,"#define %sCTX_PARAM\n",name); lineno++; + fprintf(out,"#define %sCTX_FETCH\n",name); lineno++; + fprintf(out,"#define %sCTX_STORE\n",name); lineno++; + } + if( mhflag ){ + fprintf(out,"#endif\n"); lineno++; + } + if( lemp->errsym && lemp->errsym->useCnt ){ + fprintf(out,"#define YYERRORSYMBOL %d\n",lemp->errsym->index); lineno++; + fprintf(out,"#define YYERRSYMDT yy%d\n",lemp->errsym->dtnum); lineno++; + } + if( lemp->has_fallback ){ + fprintf(out,"#define YYFALLBACK 1\n"); lineno++; + } + + /* Compute the action table, but do not output it yet. The action + ** table must be computed before generating the YYNSTATE macro because + ** we need to know how many states can be eliminated. + */ + ax = (struct axset *) calloc(lemp->nxstate*2, sizeof(ax[0])); + if( ax==0 ){ + fprintf(stderr,"malloc failed\n"); + exit(1); + } + for(i=0; i<lemp->nxstate; i++){ + stp = lemp->sorted[i]; + ax[i*2].stp = stp; + ax[i*2].isTkn = 1; + ax[i*2].nAction = stp->nTknAct; + ax[i*2+1].stp = stp; + ax[i*2+1].isTkn = 0; + ax[i*2+1].nAction = stp->nNtAct; + } + mxTknOfst = mnTknOfst = 0; + mxNtOfst = mnNtOfst = 0; + /* In an effort to minimize the action table size, use the heuristic + ** of placing the largest action sets first */ + for(i=0; i<lemp->nxstate*2; i++) ax[i].iOrder = i; + qsort(ax, lemp->nxstate*2, sizeof(ax[0]), axset_compare); + pActtab = acttab_alloc(lemp->nsymbol, lemp->nterminal); + for(i=0; i<lemp->nxstate*2 && ax[i].nAction>0; i++){ + stp = ax[i].stp; + if( ax[i].isTkn ){ + for(ap=stp->ap; ap; ap=ap->next){ + int action; + if( ap->sp->index>=lemp->nterminal ) continue; + action = compute_action(lemp, ap); + if( action<0 ) continue; + acttab_action(pActtab, ap->sp->index, action); + } + stp->iTknOfst = acttab_insert(pActtab, 1); + if( stp->iTknOfst<mnTknOfst ) mnTknOfst = stp->iTknOfst; + if( stp->iTknOfst>mxTknOfst ) mxTknOfst = stp->iTknOfst; + }else{ + for(ap=stp->ap; ap; ap=ap->next){ + int action; + if( ap->sp->index<lemp->nterminal ) continue; + if( ap->sp->index==lemp->nsymbol ) continue; + action = compute_action(lemp, ap); + if( action<0 ) continue; + acttab_action(pActtab, ap->sp->index, action); + } + stp->iNtOfst = acttab_insert(pActtab, 0); + if( stp->iNtOfst<mnNtOfst ) mnNtOfst = stp->iNtOfst; + if( stp->iNtOfst>mxNtOfst ) mxNtOfst = stp->iNtOfst; + } +#if 0 /* Uncomment for a trace of how the yy_action[] table fills out */ + { int jj, nn; + for(jj=nn=0; jj<pActtab->nAction; jj++){ + if( pActtab->aAction[jj].action<0 ) nn++; + } + printf("%4d: State %3d %s n: %2d size: %5d freespace: %d\n", + i, stp->statenum, ax[i].isTkn ? "Token" : "Var ", + ax[i].nAction, pActtab->nAction, nn); + } +#endif + } + free(ax); + + /* Mark rules that are actually used for reduce actions after all + ** optimizations have been applied + */ + for(rp=lemp->rule; rp; rp=rp->next) rp->doesReduce = LEMON_FALSE; + for(i=0; i<lemp->nxstate; i++){ + for(ap=lemp->sorted[i]->ap; ap; ap=ap->next){ + if( ap->type==REDUCE || ap->type==SHIFTREDUCE ){ + ap->x.rp->doesReduce = 1; + } + } + } + + /* Finish rendering the constants now that the action table has + ** been computed */ + fprintf(out,"#define YYNSTATE %d\n",lemp->nxstate); lineno++; + fprintf(out,"#define YYNRULE %d\n",lemp->nrule); lineno++; + fprintf(out,"#define YYNRULE_WITH_ACTION %d\n",lemp->nruleWithAction); + lineno++; + fprintf(out,"#define YYNTOKEN %d\n",lemp->nterminal); lineno++; + fprintf(out,"#define YY_MAX_SHIFT %d\n",lemp->nxstate-1); lineno++; + i = lemp->minShiftReduce; + fprintf(out,"#define YY_MIN_SHIFTREDUCE %d\n",i); lineno++; + i += lemp->nrule; + fprintf(out,"#define YY_MAX_SHIFTREDUCE %d\n", i-1); lineno++; + fprintf(out,"#define YY_ERROR_ACTION %d\n", lemp->errAction); lineno++; + fprintf(out,"#define YY_ACCEPT_ACTION %d\n", lemp->accAction); lineno++; + fprintf(out,"#define YY_NO_ACTION %d\n", lemp->noAction); lineno++; + fprintf(out,"#define YY_MIN_REDUCE %d\n", lemp->minReduce); lineno++; + i = lemp->minReduce + lemp->nrule; + fprintf(out,"#define YY_MAX_REDUCE %d\n", i-1); lineno++; + tplt_xfer(lemp->name,in,out,&lineno); + + /* Now output the action table and its associates: + ** + ** yy_action[] A single table containing all actions. + ** yy_lookahead[] A table containing the lookahead for each entry in + ** yy_action. Used to detect hash collisions. + ** yy_shift_ofst[] For each state, the offset into yy_action for + ** shifting terminals. + ** yy_reduce_ofst[] For each state, the offset into yy_action for + ** shifting non-terminals after a reduce. + ** yy_default[] Default action for each state. + */ + + /* Output the yy_action table */ + lemp->nactiontab = n = acttab_action_size(pActtab); + lemp->tablesize += n*szActionType; + fprintf(out,"#define YY_ACTTAB_COUNT (%d)\n", n); lineno++; + fprintf(out,"static const YYACTIONTYPE yy_action[] = {\n"); lineno++; + for(i=j=0; i<n; i++){ + int action = acttab_yyaction(pActtab, i); + if( action<0 ) action = lemp->noAction; + if( j==0 ) fprintf(out," /* %5d */ ", i); + fprintf(out, " %4d,", action); + if( j==9 || i==n-1 ){ + fprintf(out, "\n"); lineno++; + j = 0; + }else{ + j++; + } + } + fprintf(out, "};\n"); lineno++; + + /* Output the yy_lookahead table */ + lemp->nlookaheadtab = n = acttab_lookahead_size(pActtab); + lemp->tablesize += n*szCodeType; + fprintf(out,"static const YYCODETYPE yy_lookahead[] = {\n"); lineno++; + for(i=j=0; i<n; i++){ + int la = acttab_yylookahead(pActtab, i); + if( la<0 ) la = lemp->nsymbol; + if( j==0 ) fprintf(out," /* %5d */ ", i); + fprintf(out, " %4d,", la); + if( j==9 ){ + fprintf(out, "\n"); lineno++; + j = 0; + }else{ + j++; + } + } + /* Add extra entries to the end of the yy_lookahead[] table so that + ** yy_shift_ofst[]+iToken will always be a valid index into the array, + ** even for the largest possible value of yy_shift_ofst[] and iToken. */ + nLookAhead = lemp->nterminal + lemp->nactiontab; + while( i<nLookAhead ){ + if( j==0 ) fprintf(out," /* %5d */ ", i); + fprintf(out, " %4d,", lemp->nterminal); + if( j==9 ){ + fprintf(out, "\n"); lineno++; + j = 0; + }else{ + j++; + } + i++; + } + if( j>0 ){ fprintf(out, "\n"); lineno++; } + fprintf(out, "};\n"); lineno++; + + /* Output the yy_shift_ofst[] table */ + n = lemp->nxstate; + while( n>0 && lemp->sorted[n-1]->iTknOfst==NO_OFFSET ) n--; + fprintf(out, "#define YY_SHIFT_COUNT (%d)\n", n-1); lineno++; + fprintf(out, "#define YY_SHIFT_MIN (%d)\n", mnTknOfst); lineno++; + fprintf(out, "#define YY_SHIFT_MAX (%d)\n", mxTknOfst); lineno++; + fprintf(out, "static const %s yy_shift_ofst[] = {\n", + minimum_size_type(mnTknOfst, lemp->nterminal+lemp->nactiontab, &sz)); + lineno++; + lemp->tablesize += n*sz; + for(i=j=0; i<n; i++){ + int ofst; + stp = lemp->sorted[i]; + ofst = stp->iTknOfst; + if( ofst==NO_OFFSET ) ofst = lemp->nactiontab; + if( j==0 ) fprintf(out," /* %5d */ ", i); + fprintf(out, " %4d,", ofst); + if( j==9 || i==n-1 ){ + fprintf(out, "\n"); lineno++; + j = 0; + }else{ + j++; + } + } + fprintf(out, "};\n"); lineno++; + + /* Output the yy_reduce_ofst[] table */ + n = lemp->nxstate; + while( n>0 && lemp->sorted[n-1]->iNtOfst==NO_OFFSET ) n--; + fprintf(out, "#define YY_REDUCE_COUNT (%d)\n", n-1); lineno++; + fprintf(out, "#define YY_REDUCE_MIN (%d)\n", mnNtOfst); lineno++; + fprintf(out, "#define YY_REDUCE_MAX (%d)\n", mxNtOfst); lineno++; + fprintf(out, "static const %s yy_reduce_ofst[] = {\n", + minimum_size_type(mnNtOfst-1, mxNtOfst, &sz)); lineno++; + lemp->tablesize += n*sz; + for(i=j=0; i<n; i++){ + int ofst; + stp = lemp->sorted[i]; + ofst = stp->iNtOfst; + if( ofst==NO_OFFSET ) ofst = mnNtOfst - 1; + if( j==0 ) fprintf(out," /* %5d */ ", i); + fprintf(out, " %4d,", ofst); + if( j==9 || i==n-1 ){ + fprintf(out, "\n"); lineno++; + j = 0; + }else{ + j++; + } + } + fprintf(out, "};\n"); lineno++; + + /* Output the default action table */ + fprintf(out, "static const YYACTIONTYPE yy_default[] = {\n"); lineno++; + n = lemp->nxstate; + lemp->tablesize += n*szActionType; + for(i=j=0; i<n; i++){ + stp = lemp->sorted[i]; + if( j==0 ) fprintf(out," /* %5d */ ", i); + if( stp->iDfltReduce<0 ){ + fprintf(out, " %4d,", lemp->errAction); + }else{ + fprintf(out, " %4d,", stp->iDfltReduce + lemp->minReduce); + } + if( j==9 || i==n-1 ){ + fprintf(out, "\n"); lineno++; + j = 0; + }else{ + j++; + } + } + fprintf(out, "};\n"); lineno++; + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate the table of fallback tokens. + */ + if( lemp->has_fallback ){ + int mx = lemp->nterminal - 1; + /* 2019-08-28: Generate fallback entries for every token to avoid + ** having to do a range check on the index */ + /* while( mx>0 && lemp->symbols[mx]->fallback==0 ){ mx--; } */ + lemp->tablesize += (mx+1)*szCodeType; + for(i=0; i<=mx; i++){ + struct symbol *p = lemp->symbols[i]; + if( p->fallback==0 ){ + fprintf(out, " 0, /* %10s => nothing */\n", p->name); + }else{ + fprintf(out, " %3d, /* %10s => %s */\n", p->fallback->index, + p->name, p->fallback->name); + } + lineno++; + } + } + tplt_xfer(lemp->name, in, out, &lineno); + + /* Generate a table containing the symbolic name of every symbol + */ + for(i=0; i<lemp->nsymbol; i++){ + fprintf(out," /* %4d */ \"%s\",\n",i, lemp->symbols[i]->name); lineno++; + } + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate a table containing a text string that describes every + ** rule in the rule set of the grammar. This information is used + ** when tracing REDUCE actions. + */ + for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){ + assert( rp->iRule==i ); + fprintf(out," /* %3d */ \"", i); + writeRuleText(out, rp); + fprintf(out,"\",\n"); lineno++; + } + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate code which executes every time a symbol is popped from + ** the stack while processing errors or while destroying the parser. + ** (In other words, generate the %destructor actions) + */ + if( lemp->tokendest ){ + int once = 1; + for(i=0; i<lemp->nsymbol; i++){ + struct symbol *sp = lemp->symbols[i]; + if( sp==0 || sp->type!=TERMINAL ) continue; + if( once ){ + fprintf(out, " /* TERMINAL Destructor */\n"); lineno++; + once = 0; + } + fprintf(out," case %d: /* %s */\n", sp->index, sp->name); lineno++; + } + for(i=0; i<lemp->nsymbol && lemp->symbols[i]->type!=TERMINAL; i++); + if( i<lemp->nsymbol ){ + emit_destructor_code(out,lemp->symbols[i],lemp,&lineno); + fprintf(out," break;\n"); lineno++; + } + } + if( lemp->vardest ){ + struct symbol *dflt_sp = 0; + int once = 1; + for(i=0; i<lemp->nsymbol; i++){ + struct symbol *sp = lemp->symbols[i]; + if( sp==0 || sp->type==TERMINAL || + sp->index<=0 || sp->destructor!=0 ) continue; + if( once ){ + fprintf(out, " /* Default NON-TERMINAL Destructor */\n");lineno++; + once = 0; + } + fprintf(out," case %d: /* %s */\n", sp->index, sp->name); lineno++; + dflt_sp = sp; + } + if( dflt_sp!=0 ){ + emit_destructor_code(out,dflt_sp,lemp,&lineno); + } + fprintf(out," break;\n"); lineno++; + } + for(i=0; i<lemp->nsymbol; i++){ + struct symbol *sp = lemp->symbols[i]; + if( sp==0 || sp->type==TERMINAL || sp->destructor==0 ) continue; + if( sp->destLineno<0 ) continue; /* Already emitted */ + fprintf(out," case %d: /* %s */\n", sp->index, sp->name); lineno++; + + /* Combine duplicate destructors into a single case */ + for(j=i+1; j<lemp->nsymbol; j++){ + struct symbol *sp2 = lemp->symbols[j]; + if( sp2 && sp2->type!=TERMINAL && sp2->destructor + && sp2->dtnum==sp->dtnum + && strcmp(sp->destructor,sp2->destructor)==0 ){ + fprintf(out," case %d: /* %s */\n", + sp2->index, sp2->name); lineno++; + sp2->destLineno = -1; /* Avoid emitting this destructor again */ + } + } + + emit_destructor_code(out,lemp->symbols[i],lemp,&lineno); + fprintf(out," break;\n"); lineno++; + } + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate code which executes whenever the parser stack overflows */ + tplt_print(out,lemp,lemp->overflow,&lineno); + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate the tables of rule information. yyRuleInfoLhs[] and + ** yyRuleInfoNRhs[]. + ** + ** Note: This code depends on the fact that rules are number + ** sequentially beginning with 0. + */ + for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){ + fprintf(out," %4d, /* (%d) ", rp->lhs->index, i); + rule_print(out, rp); + fprintf(out," */\n"); lineno++; + } + tplt_xfer(lemp->name,in,out,&lineno); + for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){ + fprintf(out," %3d, /* (%d) ", -rp->nrhs, i); + rule_print(out, rp); + fprintf(out," */\n"); lineno++; + } + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate code which execution during each REDUCE action */ + i = 0; + for(rp=lemp->rule; rp; rp=rp->next){ + i += translate_code(lemp, rp); + } + if( i ){ + fprintf(out," YYMINORTYPE yylhsminor;\n"); lineno++; + } + /* First output rules other than the default: rule */ + for(rp=lemp->rule; rp; rp=rp->next){ + struct rule *rp2; /* Other rules with the same action */ + if( rp->codeEmitted ) continue; + if( rp->noCode ){ + /* No C code actions, so this will be part of the "default:" rule */ + continue; + } + fprintf(out," case %d: /* ", rp->iRule); + writeRuleText(out, rp); + fprintf(out, " */\n"); lineno++; + for(rp2=rp->next; rp2; rp2=rp2->next){ + if( rp2->code==rp->code && rp2->codePrefix==rp->codePrefix + && rp2->codeSuffix==rp->codeSuffix ){ + fprintf(out," case %d: /* ", rp2->iRule); + writeRuleText(out, rp2); + fprintf(out," */ yytestcase(yyruleno==%d);\n", rp2->iRule); lineno++; + rp2->codeEmitted = 1; + } + } + emit_code(out,rp,lemp,&lineno); + fprintf(out," break;\n"); lineno++; + rp->codeEmitted = 1; + } + /* Finally, output the default: rule. We choose as the default: all + ** empty actions. */ + fprintf(out," default:\n"); lineno++; + for(rp=lemp->rule; rp; rp=rp->next){ + if( rp->codeEmitted ) continue; + assert( rp->noCode ); + fprintf(out," /* (%d) ", rp->iRule); + writeRuleText(out, rp); + if( rp->neverReduce ){ + fprintf(out, " (NEVER REDUCES) */ assert(yyruleno!=%d);\n", + rp->iRule); lineno++; + }else if( rp->doesReduce ){ + fprintf(out, " */ yytestcase(yyruleno==%d);\n", rp->iRule); lineno++; + }else{ + fprintf(out, " (OPTIMIZED OUT) */ assert(yyruleno!=%d);\n", + rp->iRule); lineno++; + } + } + fprintf(out," break;\n"); lineno++; + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate code which executes if a parse fails */ + tplt_print(out,lemp,lemp->failure,&lineno); + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate code which executes when a syntax error occurs */ + tplt_print(out,lemp,lemp->error,&lineno); + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate code which executes when the parser accepts its input */ + tplt_print(out,lemp,lemp->accept,&lineno); + tplt_xfer(lemp->name,in,out,&lineno); + + /* Append any addition code the user desires */ + tplt_print(out,lemp,lemp->extracode,&lineno); + + acttab_free(pActtab); + fclose(in); + fclose(out); + if( sql ) fclose(sql); + return; +} + +/* Generate a header file for the parser */ +void ReportHeader(struct lemon *lemp) +{ + FILE *out, *in; + const char *prefix; + char line[LINESIZE]; + char pattern[LINESIZE]; + int i; + + if( lemp->tokenprefix ) prefix = lemp->tokenprefix; + else prefix = ""; + in = file_open(lemp,".h","rb"); + if( in ){ + int nextChar; + for(i=1; i<lemp->nterminal && fgets(line,LINESIZE,in); i++){ + lemon_sprintf(pattern,"#define %s%-30s %3d\n", + prefix,lemp->symbols[i]->name,i); + if( strcmp(line,pattern) ) break; + } + nextChar = fgetc(in); + fclose(in); + if( i==lemp->nterminal && nextChar==EOF ){ + /* No change in the file. Don't rewrite it. */ + return; + } + } + out = file_open(lemp,".h","wb"); + if( out ){ + for(i=1; i<lemp->nterminal; i++){ + fprintf(out,"#define %s%-30s %3d\n",prefix,lemp->symbols[i]->name,i); + } + fclose(out); + } + return; +} + +/* Reduce the size of the action tables, if possible, by making use +** of defaults. +** +** In this version, we take the most frequent REDUCE action and make +** it the default. Except, there is no default if the wildcard token +** is a possible look-ahead. +*/ +void CompressTables(struct lemon *lemp) +{ + struct state *stp; + struct action *ap, *ap2, *nextap; + struct rule *rp, *rp2, *rbest; + int nbest, n; + int i; + int usesWildcard; + + for(i=0; i<lemp->nstate; i++){ + stp = lemp->sorted[i]; + nbest = 0; + rbest = 0; + usesWildcard = 0; + + for(ap=stp->ap; ap; ap=ap->next){ + if( ap->type==SHIFT && ap->sp==lemp->wildcard ){ + usesWildcard = 1; + } + if( ap->type!=REDUCE ) continue; + rp = ap->x.rp; + if( rp->lhsStart ) continue; + if( rp==rbest ) continue; + n = 1; + for(ap2=ap->next; ap2; ap2=ap2->next){ + if( ap2->type!=REDUCE ) continue; + rp2 = ap2->x.rp; + if( rp2==rbest ) continue; + if( rp2==rp ) n++; + } + if( n>nbest ){ + nbest = n; + rbest = rp; + } + } + + /* Do not make a default if the number of rules to default + ** is not at least 1 or if the wildcard token is a possible + ** lookahead. + */ + if( nbest<1 || usesWildcard ) continue; + + + /* Combine matching REDUCE actions into a single default */ + for(ap=stp->ap; ap; ap=ap->next){ + if( ap->type==REDUCE && ap->x.rp==rbest ) break; + } + assert( ap ); + ap->sp = Symbol_new("{default}"); + for(ap=ap->next; ap; ap=ap->next){ + if( ap->type==REDUCE && ap->x.rp==rbest ) ap->type = NOT_USED; + } + stp->ap = Action_sort(stp->ap); + + for(ap=stp->ap; ap; ap=ap->next){ + if( ap->type==SHIFT ) break; + if( ap->type==REDUCE && ap->x.rp!=rbest ) break; + } + if( ap==0 ){ + stp->autoReduce = 1; + stp->pDfltReduce = rbest; + } + } + + /* Make a second pass over all states and actions. Convert + ** every action that is a SHIFT to an autoReduce state into + ** a SHIFTREDUCE action. + */ + for(i=0; i<lemp->nstate; i++){ + stp = lemp->sorted[i]; + for(ap=stp->ap; ap; ap=ap->next){ + struct state *pNextState; + if( ap->type!=SHIFT ) continue; + pNextState = ap->x.stp; + if( pNextState->autoReduce && pNextState->pDfltReduce!=0 ){ + ap->type = SHIFTREDUCE; + ap->x.rp = pNextState->pDfltReduce; + } + } + } + + /* If a SHIFTREDUCE action specifies a rule that has a single RHS term + ** (meaning that the SHIFTREDUCE will land back in the state where it + ** started) and if there is no C-code associated with the reduce action, + ** then we can go ahead and convert the action to be the same as the + ** action for the RHS of the rule. + */ + for(i=0; i<lemp->nstate; i++){ + stp = lemp->sorted[i]; + for(ap=stp->ap; ap; ap=nextap){ + nextap = ap->next; + if( ap->type!=SHIFTREDUCE ) continue; + rp = ap->x.rp; + if( rp->noCode==0 ) continue; + if( rp->nrhs!=1 ) continue; +#if 1 + /* Only apply this optimization to non-terminals. It would be OK to + ** apply it to terminal symbols too, but that makes the parser tables + ** larger. */ + if( ap->sp->index<lemp->nterminal ) continue; +#endif + /* If we reach this point, it means the optimization can be applied */ + nextap = ap; + for(ap2=stp->ap; ap2 && (ap2==ap || ap2->sp!=rp->lhs); ap2=ap2->next){} + assert( ap2!=0 ); + ap->spOpt = ap2->sp; + ap->type = ap2->type; + ap->x = ap2->x; + } + } +} + + +/* +** Compare two states for sorting purposes. The smaller state is the +** one with the most non-terminal actions. If they have the same number +** of non-terminal actions, then the smaller is the one with the most +** token actions. +*/ +static int stateResortCompare(const void *a, const void *b){ + const struct state *pA = *(const struct state**)a; + const struct state *pB = *(const struct state**)b; + int n; + + n = pB->nNtAct - pA->nNtAct; + if( n==0 ){ + n = pB->nTknAct - pA->nTknAct; + if( n==0 ){ + n = pB->statenum - pA->statenum; + } + } + assert( n!=0 ); + return n; +} + + +/* +** Renumber and resort states so that states with fewer choices +** occur at the end. Except, keep state 0 as the first state. +*/ +void ResortStates(struct lemon *lemp) +{ + int i; + struct state *stp; + struct action *ap; + + for(i=0; i<lemp->nstate; i++){ + stp = lemp->sorted[i]; + stp->nTknAct = stp->nNtAct = 0; + stp->iDfltReduce = -1; /* Init dflt action to "syntax error" */ + stp->iTknOfst = NO_OFFSET; + stp->iNtOfst = NO_OFFSET; + for(ap=stp->ap; ap; ap=ap->next){ + int iAction = compute_action(lemp,ap); + if( iAction>=0 ){ + if( ap->sp->index<lemp->nterminal ){ + stp->nTknAct++; + }else if( ap->sp->index<lemp->nsymbol ){ + stp->nNtAct++; + }else{ + assert( stp->autoReduce==0 || stp->pDfltReduce==ap->x.rp ); + stp->iDfltReduce = iAction; + } + } + } + } + qsort(&lemp->sorted[1], lemp->nstate-1, sizeof(lemp->sorted[0]), + stateResortCompare); + for(i=0; i<lemp->nstate; i++){ + lemp->sorted[i]->statenum = i; + } + lemp->nxstate = lemp->nstate; + while( lemp->nxstate>1 && lemp->sorted[lemp->nxstate-1]->autoReduce ){ + lemp->nxstate--; + } +} + + +/***************** From the file "set.c" ************************************/ +/* +** Set manipulation routines for the LEMON parser generator. +*/ + +static int size = 0; + +/* Set the set size */ +void SetSize(int n) +{ + size = n+1; +} + +/* Allocate a new set */ +char *SetNew(void){ + char *s; + s = (char*)calloc( size, 1); + if( s==0 ){ + memory_error(); + } + return s; +} + +/* Deallocate a set */ +void SetFree(char *s) +{ + free(s); +} + +/* Add a new element to the set. Return TRUE if the element was added +** and FALSE if it was already there. */ +int SetAdd(char *s, int e) +{ + int rv; + assert( e>=0 && e<size ); + rv = s[e]; + s[e] = 1; + return !rv; +} + +/* Add every element of s2 to s1. Return TRUE if s1 changes. */ +int SetUnion(char *s1, char *s2) +{ + int i, progress; + progress = 0; + for(i=0; i<size; i++){ + if( s2[i]==0 ) continue; + if( s1[i]==0 ){ + progress = 1; + s1[i] = 1; + } + } + return progress; +} +/********************** From the file "table.c" ****************************/ +/* +** All code in this file has been automatically generated +** from a specification in the file +** "table.q" +** by the associative array code building program "aagen". +** Do not edit this file! Instead, edit the specification +** file, then rerun aagen. +*/ +/* +** Code for processing tables in the LEMON parser generator. +*/ + +PRIVATE unsigned strhash(const char *x) +{ + unsigned h = 0; + while( *x ) h = h*13 + *(x++); + return h; +} + +/* Works like strdup, sort of. Save a string in malloced memory, but +** keep strings in a table so that the same string is not in more +** than one place. +*/ +const char *Strsafe(const char *y) +{ + const char *z; + char *cpy; + + if( y==0 ) return 0; + z = Strsafe_find(y); + if( z==0 && (cpy=(char *)malloc( lemonStrlen(y)+1 ))!=0 ){ + lemon_strcpy(cpy,y); + z = cpy; + Strsafe_insert(z); + } + MemoryCheck(z); + return z; +} + +/* There is one instance of the following structure for each +** associative array of type "x1". +*/ +struct s_x1 { + int size; /* The number of available slots. */ + /* Must be a power of 2 greater than or */ + /* equal to 1 */ + int count; /* Number of currently slots filled */ + struct s_x1node *tbl; /* The data stored here */ + struct s_x1node **ht; /* Hash table for lookups */ +}; + +/* There is one instance of this structure for every data element +** in an associative array of type "x1". +*/ +typedef struct s_x1node { + const char *data; /* The data */ + struct s_x1node *next; /* Next entry with the same hash */ + struct s_x1node **from; /* Previous link */ +} x1node; + +/* There is only one instance of the array, which is the following */ +static struct s_x1 *x1a; + +/* Allocate a new associative array */ +void Strsafe_init(void){ + if( x1a ) return; + x1a = (struct s_x1*)malloc( sizeof(struct s_x1) ); + if( x1a ){ + x1a->size = 1024; + x1a->count = 0; + x1a->tbl = (x1node*)calloc(1024, sizeof(x1node) + sizeof(x1node*)); + if( x1a->tbl==0 ){ + free(x1a); + x1a = 0; + }else{ + int i; + x1a->ht = (x1node**)&(x1a->tbl[1024]); + for(i=0; i<1024; i++) x1a->ht[i] = 0; + } + } +} +/* Insert a new record into the array. Return TRUE if successful. +** Prior data with the same key is NOT overwritten */ +int Strsafe_insert(const char *data) +{ + x1node *np; + unsigned h; + unsigned ph; + + if( x1a==0 ) return 0; + ph = strhash(data); + h = ph & (x1a->size-1); + np = x1a->ht[h]; + while( np ){ + if( strcmp(np->data,data)==0 ){ + /* An existing entry with the same key is found. */ + /* Fail because overwrite is not allows. */ + return 0; + } + np = np->next; + } + if( x1a->count>=x1a->size ){ + /* Need to make the hash table bigger */ + int i,arrSize; + struct s_x1 array; + array.size = arrSize = x1a->size*2; + array.count = x1a->count; + array.tbl = (x1node*)calloc(arrSize, sizeof(x1node) + sizeof(x1node*)); + if( array.tbl==0 ) return 0; /* Fail due to malloc failure */ + array.ht = (x1node**)&(array.tbl[arrSize]); + for(i=0; i<arrSize; i++) array.ht[i] = 0; + for(i=0; i<x1a->count; i++){ + x1node *oldnp, *newnp; + oldnp = &(x1a->tbl[i]); + h = strhash(oldnp->data) & (arrSize-1); + newnp = &(array.tbl[i]); + if( array.ht[h] ) array.ht[h]->from = &(newnp->next); + newnp->next = array.ht[h]; + newnp->data = oldnp->data; + newnp->from = &(array.ht[h]); + array.ht[h] = newnp; + } + /* free(x1a->tbl); // This program was originally for 16-bit machines. + ** Don't worry about freeing memory on modern platforms. */ + *x1a = array; + } + /* Insert the new data */ + h = ph & (x1a->size-1); + np = &(x1a->tbl[x1a->count++]); + np->data = data; + if( x1a->ht[h] ) x1a->ht[h]->from = &(np->next); + np->next = x1a->ht[h]; + x1a->ht[h] = np; + np->from = &(x1a->ht[h]); + return 1; +} + +/* Return a pointer to data assigned to the given key. Return NULL +** if no such key. */ +const char *Strsafe_find(const char *key) +{ + unsigned h; + x1node *np; + + if( x1a==0 ) return 0; + h = strhash(key) & (x1a->size-1); + np = x1a->ht[h]; + while( np ){ + if( strcmp(np->data,key)==0 ) break; + np = np->next; + } + return np ? np->data : 0; +} + +/* Return a pointer to the (terminal or nonterminal) symbol "x". +** Create a new symbol if this is the first time "x" has been seen. +*/ +struct symbol *Symbol_new(const char *x) +{ + struct symbol *sp; + + sp = Symbol_find(x); + if( sp==0 ){ + sp = (struct symbol *)calloc(1, sizeof(struct symbol) ); + MemoryCheck(sp); + sp->name = Strsafe(x); + sp->type = ISUPPER(*x) ? TERMINAL : NONTERMINAL; + sp->rule = 0; + sp->fallback = 0; + sp->prec = -1; + sp->assoc = UNK; + sp->firstset = 0; + sp->lambda = LEMON_FALSE; + sp->destructor = 0; + sp->destLineno = 0; + sp->datatype = 0; + sp->useCnt = 0; + Symbol_insert(sp,sp->name); + } + sp->useCnt++; + return sp; +} + +/* Compare two symbols for sorting purposes. Return negative, +** zero, or positive if a is less then, equal to, or greater +** than b. +** +** Symbols that begin with upper case letters (terminals or tokens) +** must sort before symbols that begin with lower case letters +** (non-terminals). And MULTITERMINAL symbols (created using the +** %token_class directive) must sort at the very end. Other than +** that, the order does not matter. +** +** We find experimentally that leaving the symbols in their original +** order (the order they appeared in the grammar file) gives the +** smallest parser tables in SQLite. +*/ +int Symbolcmpp(const void *_a, const void *_b) +{ + const struct symbol *a = *(const struct symbol **) _a; + const struct symbol *b = *(const struct symbol **) _b; + int i1 = a->type==MULTITERMINAL ? 3 : a->name[0]>'Z' ? 2 : 1; + int i2 = b->type==MULTITERMINAL ? 3 : b->name[0]>'Z' ? 2 : 1; + return i1==i2 ? a->index - b->index : i1 - i2; +} + +/* There is one instance of the following structure for each +** associative array of type "x2". +*/ +struct s_x2 { + int size; /* The number of available slots. */ + /* Must be a power of 2 greater than or */ + /* equal to 1 */ + int count; /* Number of currently slots filled */ + struct s_x2node *tbl; /* The data stored here */ + struct s_x2node **ht; /* Hash table for lookups */ +}; + +/* There is one instance of this structure for every data element +** in an associative array of type "x2". +*/ +typedef struct s_x2node { + struct symbol *data; /* The data */ + const char *key; /* The key */ + struct s_x2node *next; /* Next entry with the same hash */ + struct s_x2node **from; /* Previous link */ +} x2node; + +/* There is only one instance of the array, which is the following */ +static struct s_x2 *x2a; + +/* Allocate a new associative array */ +void Symbol_init(void){ + if( x2a ) return; + x2a = (struct s_x2*)malloc( sizeof(struct s_x2) ); + if( x2a ){ + x2a->size = 128; + x2a->count = 0; + x2a->tbl = (x2node*)calloc(128, sizeof(x2node) + sizeof(x2node*)); + if( x2a->tbl==0 ){ + free(x2a); + x2a = 0; + }else{ + int i; + x2a->ht = (x2node**)&(x2a->tbl[128]); + for(i=0; i<128; i++) x2a->ht[i] = 0; + } + } +} +/* Insert a new record into the array. Return TRUE if successful. +** Prior data with the same key is NOT overwritten */ +int Symbol_insert(struct symbol *data, const char *key) +{ + x2node *np; + unsigned h; + unsigned ph; + + if( x2a==0 ) return 0; + ph = strhash(key); + h = ph & (x2a->size-1); + np = x2a->ht[h]; + while( np ){ + if( strcmp(np->key,key)==0 ){ + /* An existing entry with the same key is found. */ + /* Fail because overwrite is not allows. */ + return 0; + } + np = np->next; + } + if( x2a->count>=x2a->size ){ + /* Need to make the hash table bigger */ + int i,arrSize; + struct s_x2 array; + array.size = arrSize = x2a->size*2; + array.count = x2a->count; + array.tbl = (x2node*)calloc(arrSize, sizeof(x2node) + sizeof(x2node*)); + if( array.tbl==0 ) return 0; /* Fail due to malloc failure */ + array.ht = (x2node**)&(array.tbl[arrSize]); + for(i=0; i<arrSize; i++) array.ht[i] = 0; + for(i=0; i<x2a->count; i++){ + x2node *oldnp, *newnp; + oldnp = &(x2a->tbl[i]); + h = strhash(oldnp->key) & (arrSize-1); + newnp = &(array.tbl[i]); + if( array.ht[h] ) array.ht[h]->from = &(newnp->next); + newnp->next = array.ht[h]; + newnp->key = oldnp->key; + newnp->data = oldnp->data; + newnp->from = &(array.ht[h]); + array.ht[h] = newnp; + } + /* free(x2a->tbl); // This program was originally written for 16-bit + ** machines. Don't worry about freeing this trivial amount of memory + ** on modern platforms. Just leak it. */ + *x2a = array; + } + /* Insert the new data */ + h = ph & (x2a->size-1); + np = &(x2a->tbl[x2a->count++]); + np->key = key; + np->data = data; + if( x2a->ht[h] ) x2a->ht[h]->from = &(np->next); + np->next = x2a->ht[h]; + x2a->ht[h] = np; + np->from = &(x2a->ht[h]); + return 1; +} + +/* Return a pointer to data assigned to the given key. Return NULL +** if no such key. */ +struct symbol *Symbol_find(const char *key) +{ + unsigned h; + x2node *np; + + if( x2a==0 ) return 0; + h = strhash(key) & (x2a->size-1); + np = x2a->ht[h]; + while( np ){ + if( strcmp(np->key,key)==0 ) break; + np = np->next; + } + return np ? np->data : 0; +} + +/* Return the n-th data. Return NULL if n is out of range. */ +struct symbol *Symbol_Nth(int n) +{ + struct symbol *data; + if( x2a && n>0 && n<=x2a->count ){ + data = x2a->tbl[n-1].data; + }else{ + data = 0; + } + return data; +} + +/* Return the size of the array */ +int Symbol_count() +{ + return x2a ? x2a->count : 0; +} + +/* Return an array of pointers to all data in the table. +** The array is obtained from malloc. Return NULL if memory allocation +** problems, or if the array is empty. */ +struct symbol **Symbol_arrayof() +{ + struct symbol **array; + int i,arrSize; + if( x2a==0 ) return 0; + arrSize = x2a->count; + array = (struct symbol **)calloc(arrSize, sizeof(struct symbol *)); + if( array ){ + for(i=0; i<arrSize; i++) array[i] = x2a->tbl[i].data; + } + return array; +} + +/* Compare two configurations */ +int Configcmp(const char *_a,const char *_b) +{ + const struct config *a = (struct config *) _a; + const struct config *b = (struct config *) _b; + int x; + x = a->rp->index - b->rp->index; + if( x==0 ) x = a->dot - b->dot; + return x; +} + +/* Compare two states */ +PRIVATE int statecmp(struct config *a, struct config *b) +{ + int rc; + for(rc=0; rc==0 && a && b; a=a->bp, b=b->bp){ + rc = a->rp->index - b->rp->index; + if( rc==0 ) rc = a->dot - b->dot; + } + if( rc==0 ){ + if( a ) rc = 1; + if( b ) rc = -1; + } + return rc; +} + +/* Hash a state */ +PRIVATE unsigned statehash(struct config *a) +{ + unsigned h=0; + while( a ){ + h = h*571 + a->rp->index*37 + a->dot; + a = a->bp; + } + return h; +} + +/* Allocate a new state structure */ +struct state *State_new() +{ + struct state *newstate; + newstate = (struct state *)calloc(1, sizeof(struct state) ); + MemoryCheck(newstate); + return newstate; +} + +/* There is one instance of the following structure for each +** associative array of type "x3". +*/ +struct s_x3 { + int size; /* The number of available slots. */ + /* Must be a power of 2 greater than or */ + /* equal to 1 */ + int count; /* Number of currently slots filled */ + struct s_x3node *tbl; /* The data stored here */ + struct s_x3node **ht; /* Hash table for lookups */ +}; + +/* There is one instance of this structure for every data element +** in an associative array of type "x3". +*/ +typedef struct s_x3node { + struct state *data; /* The data */ + struct config *key; /* The key */ + struct s_x3node *next; /* Next entry with the same hash */ + struct s_x3node **from; /* Previous link */ +} x3node; + +/* There is only one instance of the array, which is the following */ +static struct s_x3 *x3a; + +/* Allocate a new associative array */ +void State_init(void){ + if( x3a ) return; + x3a = (struct s_x3*)malloc( sizeof(struct s_x3) ); + if( x3a ){ + x3a->size = 128; + x3a->count = 0; + x3a->tbl = (x3node*)calloc(128, sizeof(x3node) + sizeof(x3node*)); + if( x3a->tbl==0 ){ + free(x3a); + x3a = 0; + }else{ + int i; + x3a->ht = (x3node**)&(x3a->tbl[128]); + for(i=0; i<128; i++) x3a->ht[i] = 0; + } + } +} +/* Insert a new record into the array. Return TRUE if successful. +** Prior data with the same key is NOT overwritten */ +int State_insert(struct state *data, struct config *key) +{ + x3node *np; + unsigned h; + unsigned ph; + + if( x3a==0 ) return 0; + ph = statehash(key); + h = ph & (x3a->size-1); + np = x3a->ht[h]; + while( np ){ + if( statecmp(np->key,key)==0 ){ + /* An existing entry with the same key is found. */ + /* Fail because overwrite is not allows. */ + return 0; + } + np = np->next; + } + if( x3a->count>=x3a->size ){ + /* Need to make the hash table bigger */ + int i,arrSize; + struct s_x3 array; + array.size = arrSize = x3a->size*2; + array.count = x3a->count; + array.tbl = (x3node*)calloc(arrSize, sizeof(x3node) + sizeof(x3node*)); + if( array.tbl==0 ) return 0; /* Fail due to malloc failure */ + array.ht = (x3node**)&(array.tbl[arrSize]); + for(i=0; i<arrSize; i++) array.ht[i] = 0; + for(i=0; i<x3a->count; i++){ + x3node *oldnp, *newnp; + oldnp = &(x3a->tbl[i]); + h = statehash(oldnp->key) & (arrSize-1); + newnp = &(array.tbl[i]); + if( array.ht[h] ) array.ht[h]->from = &(newnp->next); + newnp->next = array.ht[h]; + newnp->key = oldnp->key; + newnp->data = oldnp->data; + newnp->from = &(array.ht[h]); + array.ht[h] = newnp; + } + free(x3a->tbl); + *x3a = array; + } + /* Insert the new data */ + h = ph & (x3a->size-1); + np = &(x3a->tbl[x3a->count++]); + np->key = key; + np->data = data; + if( x3a->ht[h] ) x3a->ht[h]->from = &(np->next); + np->next = x3a->ht[h]; + x3a->ht[h] = np; + np->from = &(x3a->ht[h]); + return 1; +} + +/* Return a pointer to data assigned to the given key. Return NULL +** if no such key. */ +struct state *State_find(struct config *key) +{ + unsigned h; + x3node *np; + + if( x3a==0 ) return 0; + h = statehash(key) & (x3a->size-1); + np = x3a->ht[h]; + while( np ){ + if( statecmp(np->key,key)==0 ) break; + np = np->next; + } + return np ? np->data : 0; +} + +/* Return an array of pointers to all data in the table. +** The array is obtained from malloc. Return NULL if memory allocation +** problems, or if the array is empty. */ +struct state **State_arrayof(void) +{ + struct state **array; + int i,arrSize; + if( x3a==0 ) return 0; + arrSize = x3a->count; + array = (struct state **)calloc(arrSize, sizeof(struct state *)); + if( array ){ + for(i=0; i<arrSize; i++) array[i] = x3a->tbl[i].data; + } + return array; +} + +/* Hash a configuration */ +PRIVATE unsigned confighash(struct config *a) +{ + unsigned h=0; + h = h*571 + a->rp->index*37 + a->dot; + return h; +} + +/* There is one instance of the following structure for each +** associative array of type "x4". +*/ +struct s_x4 { + int size; /* The number of available slots. */ + /* Must be a power of 2 greater than or */ + /* equal to 1 */ + int count; /* Number of currently slots filled */ + struct s_x4node *tbl; /* The data stored here */ + struct s_x4node **ht; /* Hash table for lookups */ +}; + +/* There is one instance of this structure for every data element +** in an associative array of type "x4". +*/ +typedef struct s_x4node { + struct config *data; /* The data */ + struct s_x4node *next; /* Next entry with the same hash */ + struct s_x4node **from; /* Previous link */ +} x4node; + +/* There is only one instance of the array, which is the following */ +static struct s_x4 *x4a; + +/* Allocate a new associative array */ +void Configtable_init(void){ + if( x4a ) return; + x4a = (struct s_x4*)malloc( sizeof(struct s_x4) ); + if( x4a ){ + x4a->size = 64; + x4a->count = 0; + x4a->tbl = (x4node*)calloc(64, sizeof(x4node) + sizeof(x4node*)); + if( x4a->tbl==0 ){ + free(x4a); + x4a = 0; + }else{ + int i; + x4a->ht = (x4node**)&(x4a->tbl[64]); + for(i=0; i<64; i++) x4a->ht[i] = 0; + } + } +} +/* Insert a new record into the array. Return TRUE if successful. +** Prior data with the same key is NOT overwritten */ +int Configtable_insert(struct config *data) +{ + x4node *np; + unsigned h; + unsigned ph; + + if( x4a==0 ) return 0; + ph = confighash(data); + h = ph & (x4a->size-1); + np = x4a->ht[h]; + while( np ){ + if( Configcmp((const char *) np->data,(const char *) data)==0 ){ + /* An existing entry with the same key is found. */ + /* Fail because overwrite is not allows. */ + return 0; + } + np = np->next; + } + if( x4a->count>=x4a->size ){ + /* Need to make the hash table bigger */ + int i,arrSize; + struct s_x4 array; + array.size = arrSize = x4a->size*2; + array.count = x4a->count; + array.tbl = (x4node*)calloc(arrSize, sizeof(x4node) + sizeof(x4node*)); + if( array.tbl==0 ) return 0; /* Fail due to malloc failure */ + array.ht = (x4node**)&(array.tbl[arrSize]); + for(i=0; i<arrSize; i++) array.ht[i] = 0; + for(i=0; i<x4a->count; i++){ + x4node *oldnp, *newnp; + oldnp = &(x4a->tbl[i]); + h = confighash(oldnp->data) & (arrSize-1); + newnp = &(array.tbl[i]); + if( array.ht[h] ) array.ht[h]->from = &(newnp->next); + newnp->next = array.ht[h]; + newnp->data = oldnp->data; + newnp->from = &(array.ht[h]); + array.ht[h] = newnp; + } + /* free(x4a->tbl); // This code was originall written for 16-bit machines. + ** on modern machines, don't worry about freeing this trival amount of + ** memory. */ + *x4a = array; + } + /* Insert the new data */ + h = ph & (x4a->size-1); + np = &(x4a->tbl[x4a->count++]); + np->data = data; + if( x4a->ht[h] ) x4a->ht[h]->from = &(np->next); + np->next = x4a->ht[h]; + x4a->ht[h] = np; + np->from = &(x4a->ht[h]); + return 1; +} + +/* Return a pointer to data assigned to the given key. Return NULL +** if no such key. */ +struct config *Configtable_find(struct config *key) +{ + int h; + x4node *np; + + if( x4a==0 ) return 0; + h = confighash(key) & (x4a->size-1); + np = x4a->ht[h]; + while( np ){ + if( Configcmp((const char *) np->data,(const char *) key)==0 ) break; + np = np->next; + } + return np ? np->data : 0; +} + +/* Remove all data from the table. Pass each data to the function "f" +** as it is removed. ("f" may be null to avoid this step.) */ +void Configtable_clear(int(*f)(struct config *)) +{ + int i; + if( x4a==0 || x4a->count==0 ) return; + if( f ) for(i=0; i<x4a->count; i++) (*f)(x4a->tbl[i].data); + for(i=0; i<x4a->size; i++) x4a->ht[i] = 0; + x4a->count = 0; + return; +} diff --git a/tool/lempar.c b/tool/lempar.c new file mode 100644 index 0000000..8cc5789 --- /dev/null +++ b/tool/lempar.c @@ -0,0 +1,1068 @@ +/* +** 2000-05-29 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Driver template for the LEMON parser generator. +** +** The "lemon" program processes an LALR(1) input grammar file, then uses +** this template to construct a parser. The "lemon" program inserts text +** at each "%%" line. Also, any "P-a-r-s-e" identifer prefix (without the +** interstitial "-" characters) contained in this template is changed into +** the value of the %name directive from the grammar. Otherwise, the content +** of this template is copied straight through into the generate parser +** source file. +** +** The following is the concatenation of all %include directives from the +** input grammar file: +*/ +/************ Begin %include sections from the grammar ************************/ +%% +/**************** End of %include directives **********************************/ +/* These constants specify the various numeric values for terminal symbols. +***************** Begin token definitions *************************************/ +%% +/**************** End token definitions ***************************************/ + +/* The next sections is a series of control #defines. +** various aspects of the generated parser. +** YYCODETYPE is the data type used to store the integer codes +** that represent terminal and non-terminal symbols. +** "unsigned char" is used if there are fewer than +** 256 symbols. Larger types otherwise. +** YYNOCODE is a number of type YYCODETYPE that is not used for +** any terminal or nonterminal symbol. +** YYFALLBACK If defined, this indicates that one or more tokens +** (also known as: "terminal symbols") have fall-back +** values which should be used if the original symbol +** would not parse. This permits keywords to sometimes +** be used as identifiers, for example. +** YYACTIONTYPE is the data type used for "action codes" - numbers +** that indicate what to do in response to the next +** token. +** ParseTOKENTYPE is the data type used for minor type for terminal +** symbols. Background: A "minor type" is a semantic +** value associated with a terminal or non-terminal +** symbols. For example, for an "ID" terminal symbol, +** the minor type might be the name of the identifier. +** Each non-terminal can have a different minor type. +** Terminal symbols all have the same minor type, though. +** This macros defines the minor type for terminal +** symbols. +** YYMINORTYPE is the data type used for all minor types. +** This is typically a union of many types, one of +** which is ParseTOKENTYPE. The entry in the union +** for terminal symbols is called "yy0". +** YYSTACKDEPTH is the maximum depth of the parser's stack. If +** zero the stack is dynamically sized using realloc() +** ParseARG_SDECL A static variable declaration for the %extra_argument +** ParseARG_PDECL A parameter declaration for the %extra_argument +** ParseARG_PARAM Code to pass %extra_argument as a subroutine parameter +** ParseARG_STORE Code to store %extra_argument into yypParser +** ParseARG_FETCH Code to extract %extra_argument from yypParser +** ParseCTX_* As ParseARG_ except for %extra_context +** YYERRORSYMBOL is the code number of the error symbol. If not +** defined, then do no error processing. +** YYNSTATE the combined number of states. +** YYNRULE the number of rules in the grammar +** YYNTOKEN Number of terminal symbols +** YY_MAX_SHIFT Maximum value for shift actions +** YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions +** YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions +** YY_ERROR_ACTION The yy_action[] code for syntax error +** YY_ACCEPT_ACTION The yy_action[] code for accept +** YY_NO_ACTION The yy_action[] code for no-op +** YY_MIN_REDUCE Minimum value for reduce actions +** YY_MAX_REDUCE Maximum value for reduce actions +*/ +#ifndef INTERFACE +# define INTERFACE 1 +#endif +/************* Begin control #defines *****************************************/ +%% +/************* End control #defines *******************************************/ +#define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) + +/* Define the yytestcase() macro to be a no-op if is not already defined +** otherwise. +** +** Applications can choose to define yytestcase() in the %include section +** to a macro that can assist in verifying code coverage. For production +** code the yytestcase() macro should be turned off. But it is useful +** for testing. +*/ +#ifndef yytestcase +# define yytestcase(X) +#endif + + +/* Next are the tables used to determine what action to take based on the +** current state and lookahead token. These tables are used to implement +** functions that take a state number and lookahead value and return an +** action integer. +** +** Suppose the action integer is N. Then the action is determined as +** follows +** +** 0 <= N <= YY_MAX_SHIFT Shift N. That is, push the lookahead +** token onto the stack and goto state N. +** +** N between YY_MIN_SHIFTREDUCE Shift to an arbitrary state then +** and YY_MAX_SHIFTREDUCE reduce by rule N-YY_MIN_SHIFTREDUCE. +** +** N == YY_ERROR_ACTION A syntax error has occurred. +** +** N == YY_ACCEPT_ACTION The parser accepts its input. +** +** N == YY_NO_ACTION No such action. Denotes unused +** slots in the yy_action[] table. +** +** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE +** and YY_MAX_REDUCE +** +** The action table is constructed as a single large table named yy_action[]. +** Given state S and lookahead X, the action is computed as either: +** +** (A) N = yy_action[ yy_shift_ofst[S] + X ] +** (B) N = yy_default[S] +** +** The (A) formula is preferred. The B formula is used instead if +** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X. +** +** The formulas above are for computing the action when the lookahead is +** a terminal symbol. If the lookahead is a non-terminal (as occurs after +** a reduce action) then the yy_reduce_ofst[] array is used in place of +** the yy_shift_ofst[] array. +** +** The following are the tables generated in this section: +** +** yy_action[] A single table containing all actions. +** yy_lookahead[] A table containing the lookahead for each entry in +** yy_action. Used to detect hash collisions. +** yy_shift_ofst[] For each state, the offset into yy_action for +** shifting terminals. +** yy_reduce_ofst[] For each state, the offset into yy_action for +** shifting non-terminals after a reduce. +** yy_default[] Default action for each state. +** +*********** Begin parsing tables **********************************************/ +%% +/********** End of lemon-generated parsing tables *****************************/ + +/* The next table maps tokens (terminal symbols) into fallback tokens. +** If a construct like the following: +** +** %fallback ID X Y Z. +** +** appears in the grammar, then ID becomes a fallback token for X, Y, +** and Z. Whenever one of the tokens X, Y, or Z is input to the parser +** but it does not parse, the type of the token is changed to ID and +** the parse is retried before an error is thrown. +** +** This feature can be used, for example, to cause some keywords in a language +** to revert to identifiers if they keyword does not apply in the context where +** it appears. +*/ +#ifdef YYFALLBACK +static const YYCODETYPE yyFallback[] = { +%% +}; +#endif /* YYFALLBACK */ + +/* The following structure represents a single element of the +** parser's stack. Information stored includes: +** +** + The state number for the parser at this level of the stack. +** +** + The value of the token stored at this level of the stack. +** (In other words, the "major" token.) +** +** + The semantic value stored at this level of the stack. This is +** the information used by the action routines in the grammar. +** It is sometimes called the "minor" token. +** +** After the "shift" half of a SHIFTREDUCE action, the stateno field +** actually contains the reduce action for the second half of the +** SHIFTREDUCE. +*/ +struct yyStackEntry { + YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */ + YYCODETYPE major; /* The major token value. This is the code + ** number for the token at this stack level */ + YYMINORTYPE minor; /* The user-supplied minor token value. This + ** is the value of the token */ +}; +typedef struct yyStackEntry yyStackEntry; + +/* The state of the parser is completely contained in an instance of +** the following structure */ +struct yyParser { + yyStackEntry *yytos; /* Pointer to top element of the stack */ +#ifdef YYTRACKMAXSTACKDEPTH + int yyhwm; /* High-water mark of the stack */ +#endif +#ifndef YYNOERRORRECOVERY + int yyerrcnt; /* Shifts left before out of the error */ +#endif + ParseARG_SDECL /* A place to hold %extra_argument */ + ParseCTX_SDECL /* A place to hold %extra_context */ +#if YYSTACKDEPTH<=0 + int yystksz; /* Current side of the stack */ + yyStackEntry *yystack; /* The parser's stack */ + yyStackEntry yystk0; /* First stack entry */ +#else + yyStackEntry yystack[YYSTACKDEPTH]; /* The parser's stack */ + yyStackEntry *yystackEnd; /* Last entry in the stack */ +#endif +}; +typedef struct yyParser yyParser; + +#include <assert.h> +#ifndef NDEBUG +#include <stdio.h> +static FILE *yyTraceFILE = 0; +static char *yyTracePrompt = 0; +#endif /* NDEBUG */ + +#ifndef NDEBUG +/* +** Turn parser tracing on by giving a stream to which to write the trace +** and a prompt to preface each trace message. Tracing is turned off +** by making either argument NULL +** +** Inputs: +** <ul> +** <li> A FILE* to which trace output should be written. +** If NULL, then tracing is turned off. +** <li> A prefix string written at the beginning of every +** line of trace output. If NULL, then tracing is +** turned off. +** </ul> +** +** Outputs: +** None. +*/ +void ParseTrace(FILE *TraceFILE, char *zTracePrompt){ + yyTraceFILE = TraceFILE; + yyTracePrompt = zTracePrompt; + if( yyTraceFILE==0 ) yyTracePrompt = 0; + else if( yyTracePrompt==0 ) yyTraceFILE = 0; +} +#endif /* NDEBUG */ + +#if defined(YYCOVERAGE) || !defined(NDEBUG) +/* For tracing shifts, the names of all terminals and nonterminals +** are required. The following table supplies these names */ +static const char *const yyTokenName[] = { +%% +}; +#endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ + +#ifndef NDEBUG +/* For tracing reduce actions, the names of all rules are required. +*/ +static const char *const yyRuleName[] = { +%% +}; +#endif /* NDEBUG */ + + +#if YYSTACKDEPTH<=0 +/* +** Try to increase the size of the parser stack. Return the number +** of errors. Return 0 on success. +*/ +static int yyGrowStack(yyParser *p){ + int newSize; + int idx; + yyStackEntry *pNew; + + newSize = p->yystksz*2 + 100; + idx = p->yytos ? (int)(p->yytos - p->yystack) : 0; + if( p->yystack==&p->yystk0 ){ + pNew = malloc(newSize*sizeof(pNew[0])); + if( pNew ) pNew[0] = p->yystk0; + }else{ + pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); + } + if( pNew ){ + p->yystack = pNew; + p->yytos = &p->yystack[idx]; +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n", + yyTracePrompt, p->yystksz, newSize); + } +#endif + p->yystksz = newSize; + } + return pNew==0; +} +#endif + +/* Datatype of the argument to the memory allocated passed as the +** second argument to ParseAlloc() below. This can be changed by +** putting an appropriate #define in the %include section of the input +** grammar. +*/ +#ifndef YYMALLOCARGTYPE +# define YYMALLOCARGTYPE size_t +#endif + +/* Initialize a new parser that has already been allocated. +*/ +void ParseInit(void *yypRawParser ParseCTX_PDECL){ + yyParser *yypParser = (yyParser*)yypRawParser; + ParseCTX_STORE +#ifdef YYTRACKMAXSTACKDEPTH + yypParser->yyhwm = 0; +#endif +#if YYSTACKDEPTH<=0 + yypParser->yytos = NULL; + yypParser->yystack = NULL; + yypParser->yystksz = 0; + if( yyGrowStack(yypParser) ){ + yypParser->yystack = &yypParser->yystk0; + yypParser->yystksz = 1; + } +#endif +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + yypParser->yytos = yypParser->yystack; + yypParser->yystack[0].stateno = 0; + yypParser->yystack[0].major = 0; +#if YYSTACKDEPTH>0 + yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; +#endif +} + +#ifndef Parse_ENGINEALWAYSONSTACK +/* +** This function allocates a new parser. +** The only argument is a pointer to a function which works like +** malloc. +** +** Inputs: +** A pointer to the function used to allocate memory. +** +** Outputs: +** A pointer to a parser. This pointer is used in subsequent calls +** to Parse and ParseFree. +*/ +void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE) ParseCTX_PDECL){ + yyParser *yypParser; + yypParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); + if( yypParser ){ + ParseCTX_STORE + ParseInit(yypParser ParseCTX_PARAM); + } + return (void*)yypParser; +} +#endif /* Parse_ENGINEALWAYSONSTACK */ + + +/* The following function deletes the "minor type" or semantic value +** associated with a symbol. The symbol can be either a terminal +** or nonterminal. "yymajor" is the symbol code, and "yypminor" is +** a pointer to the value to be deleted. The code used to do the +** deletions is derived from the %destructor and/or %token_destructor +** directives of the input grammar. +*/ +static void yy_destructor( + yyParser *yypParser, /* The parser */ + YYCODETYPE yymajor, /* Type code for object to destroy */ + YYMINORTYPE *yypminor /* The object to be destroyed */ +){ + ParseARG_FETCH + ParseCTX_FETCH + switch( yymajor ){ + /* Here is inserted the actions which take place when a + ** terminal or non-terminal is destroyed. This can happen + ** when the symbol is popped from the stack during a + ** reduce or during error processing or when a parser is + ** being destroyed before it is finished parsing. + ** + ** Note: during a reduce, the only symbols destroyed are those + ** which appear on the RHS of the rule, but which are *not* used + ** inside the C code. + */ +/********* Begin destructor definitions ***************************************/ +%% +/********* End destructor definitions *****************************************/ + default: break; /* If no destructor action specified: do nothing */ + } +} + +/* +** Pop the parser's stack once. +** +** If there is a destructor routine associated with the token which +** is popped from the stack, then call it. +*/ +static void yy_pop_parser_stack(yyParser *pParser){ + yyStackEntry *yytos; + assert( pParser->yytos!=0 ); + assert( pParser->yytos > pParser->yystack ); + yytos = pParser->yytos--; +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sPopping %s\n", + yyTracePrompt, + yyTokenName[yytos->major]); + } +#endif + yy_destructor(pParser, yytos->major, &yytos->minor); +} + +/* +** Clear all secondary memory allocations from the parser +*/ +void ParseFinalize(void *p){ + yyParser *pParser = (yyParser*)p; + while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser); +#if YYSTACKDEPTH<=0 + if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack); +#endif +} + +#ifndef Parse_ENGINEALWAYSONSTACK +/* +** Deallocate and destroy a parser. Destructors are called for +** all stack elements before shutting the parser down. +** +** If the YYPARSEFREENEVERNULL macro exists (for example because it +** is defined in a %include section of the input grammar) then it is +** assumed that the input pointer is never NULL. +*/ +void ParseFree( + void *p, /* The parser to be deleted */ + void (*freeProc)(void*) /* Function used to reclaim memory */ +){ +#ifndef YYPARSEFREENEVERNULL + if( p==0 ) return; +#endif + ParseFinalize(p); + (*freeProc)(p); +} +#endif /* Parse_ENGINEALWAYSONSTACK */ + +/* +** Return the peak depth of the stack for a parser. +*/ +#ifdef YYTRACKMAXSTACKDEPTH +int ParseStackPeak(void *p){ + yyParser *pParser = (yyParser*)p; + return pParser->yyhwm; +} +#endif + +/* This array of booleans keeps track of the parser statement +** coverage. The element yycoverage[X][Y] is set when the parser +** is in state X and has a lookahead token Y. In a well-tested +** systems, every element of this matrix should end up being set. +*/ +#if defined(YYCOVERAGE) +static unsigned char yycoverage[YYNSTATE][YYNTOKEN]; +#endif + +/* +** Write into out a description of every state/lookahead combination that +** +** (1) has not been used by the parser, and +** (2) is not a syntax error. +** +** Return the number of missed state/lookahead combinations. +*/ +#if defined(YYCOVERAGE) +int ParseCoverage(FILE *out){ + int stateno, iLookAhead, i; + int nMissed = 0; + for(stateno=0; stateno<YYNSTATE; stateno++){ + i = yy_shift_ofst[stateno]; + for(iLookAhead=0; iLookAhead<YYNTOKEN; iLookAhead++){ + if( yy_lookahead[i+iLookAhead]!=iLookAhead ) continue; + if( yycoverage[stateno][iLookAhead]==0 ) nMissed++; + if( out ){ + fprintf(out,"State %d lookahead %s %s\n", stateno, + yyTokenName[iLookAhead], + yycoverage[stateno][iLookAhead] ? "ok" : "missed"); + } + } + } + return nMissed; +} +#endif + +/* +** Find the appropriate action for a parser given the terminal +** look-ahead token iLookAhead. +*/ +static YYACTIONTYPE yy_find_shift_action( + YYCODETYPE iLookAhead, /* The look-ahead token */ + YYACTIONTYPE stateno /* Current state number */ +){ + int i; + + if( stateno>YY_MAX_SHIFT ) return stateno; + assert( stateno <= YY_SHIFT_COUNT ); +#if defined(YYCOVERAGE) + yycoverage[stateno][iLookAhead] = 1; +#endif + do{ + i = yy_shift_ofst[stateno]; + assert( i>=0 ); + assert( i<=YY_ACTTAB_COUNT ); + assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); + assert( iLookAhead!=YYNOCODE ); + assert( iLookAhead < YYNTOKEN ); + i += iLookAhead; + assert( i<(int)YY_NLOOKAHEAD ); + if( yy_lookahead[i]!=iLookAhead ){ +#ifdef YYFALLBACK + YYCODETYPE iFallback; /* Fallback token */ + assert( iLookAhead<sizeof(yyFallback)/sizeof(yyFallback[0]) ); + iFallback = yyFallback[iLookAhead]; + if( iFallback!=0 ){ +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE, "%sFALLBACK %s => %s\n", + yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]); + } +#endif + assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */ + iLookAhead = iFallback; + continue; + } +#endif +#ifdef YYWILDCARD + { + int j = i - iLookAhead + YYWILDCARD; + assert( j<(int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0])) ); + if( yy_lookahead[j]==YYWILDCARD && iLookAhead>0 ){ +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", + yyTracePrompt, yyTokenName[iLookAhead], + yyTokenName[YYWILDCARD]); + } +#endif /* NDEBUG */ + return yy_action[j]; + } + } +#endif /* YYWILDCARD */ + return yy_default[stateno]; + }else{ + assert( i>=0 && i<(int)(sizeof(yy_action)/sizeof(yy_action[0])) ); + return yy_action[i]; + } + }while(1); +} + +/* +** Find the appropriate action for a parser given the non-terminal +** look-ahead token iLookAhead. +*/ +static YYACTIONTYPE yy_find_reduce_action( + YYACTIONTYPE stateno, /* Current state number */ + YYCODETYPE iLookAhead /* The look-ahead token */ +){ + int i; +#ifdef YYERRORSYMBOL + if( stateno>YY_REDUCE_COUNT ){ + return yy_default[stateno]; + } +#else + assert( stateno<=YY_REDUCE_COUNT ); +#endif + i = yy_reduce_ofst[stateno]; + assert( iLookAhead!=YYNOCODE ); + i += iLookAhead; +#ifdef YYERRORSYMBOL + if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ + return yy_default[stateno]; + } +#else + assert( i>=0 && i<YY_ACTTAB_COUNT ); + assert( yy_lookahead[i]==iLookAhead ); +#endif + return yy_action[i]; +} + +/* +** The following routine is called if the stack overflows. +*/ +static void yyStackOverflow(yyParser *yypParser){ + ParseARG_FETCH + ParseCTX_FETCH +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sStack Overflow!\n",yyTracePrompt); + } +#endif + while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); + /* Here code is inserted which will execute if the parser + ** stack every overflows */ +/******** Begin %stack_overflow code ******************************************/ +%% +/******** End %stack_overflow code ********************************************/ + ParseARG_STORE /* Suppress warning about unused %extra_argument var */ + ParseCTX_STORE +} + +/* +** Print tracing information for a SHIFT action +*/ +#ifndef NDEBUG +static void yyTraceShift(yyParser *yypParser, int yyNewState, const char *zTag){ + if( yyTraceFILE ){ + if( yyNewState<YYNSTATE ){ + fprintf(yyTraceFILE,"%s%s '%s', go to state %d\n", + yyTracePrompt, zTag, yyTokenName[yypParser->yytos->major], + yyNewState); + }else{ + fprintf(yyTraceFILE,"%s%s '%s', pending reduce %d\n", + yyTracePrompt, zTag, yyTokenName[yypParser->yytos->major], + yyNewState - YY_MIN_REDUCE); + } + } +} +#else +# define yyTraceShift(X,Y,Z) +#endif + +/* +** Perform a shift action. +*/ +static void yy_shift( + yyParser *yypParser, /* The parser to be shifted */ + YYACTIONTYPE yyNewState, /* The new state to shift in */ + YYCODETYPE yyMajor, /* The major token to shift in */ + ParseTOKENTYPE yyMinor /* The minor token to shift in */ +){ + yyStackEntry *yytos; + yypParser->yytos++; +#ifdef YYTRACKMAXSTACKDEPTH + if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ + yypParser->yyhwm++; + assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) ); + } +#endif +#if YYSTACKDEPTH>0 + if( yypParser->yytos>yypParser->yystackEnd ){ + yypParser->yytos--; + yyStackOverflow(yypParser); + return; + } +#else + if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz] ){ + if( yyGrowStack(yypParser) ){ + yypParser->yytos--; + yyStackOverflow(yypParser); + return; + } + } +#endif + if( yyNewState > YY_MAX_SHIFT ){ + yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; + } + yytos = yypParser->yytos; + yytos->stateno = yyNewState; + yytos->major = yyMajor; + yytos->minor.yy0 = yyMinor; + yyTraceShift(yypParser, yyNewState, "Shift"); +} + +/* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side +** of that rule */ +static const YYCODETYPE yyRuleInfoLhs[] = { +%% +}; + +/* For rule J, yyRuleInfoNRhs[J] contains the negative of the number +** of symbols on the right-hand side of that rule. */ +static const signed char yyRuleInfoNRhs[] = { +%% +}; + +static void yy_accept(yyParser*); /* Forward Declaration */ + +/* +** Perform a reduce action and the shift that must immediately +** follow the reduce. +** +** The yyLookahead and yyLookaheadToken parameters provide reduce actions +** access to the lookahead token (if any). The yyLookahead will be YYNOCODE +** if the lookahead token has already been consumed. As this procedure is +** only called from one place, optimizing compilers will in-line it, which +** means that the extra parameters have no performance impact. +*/ +static YYACTIONTYPE yy_reduce( + yyParser *yypParser, /* The parser */ + unsigned int yyruleno, /* Number of the rule by which to reduce */ + int yyLookahead, /* Lookahead token, or YYNOCODE if none */ + ParseTOKENTYPE yyLookaheadToken /* Value of the lookahead token */ + ParseCTX_PDECL /* %extra_context */ +){ + int yygoto; /* The next state */ + YYACTIONTYPE yyact; /* The next action */ + yyStackEntry *yymsp; /* The top of the parser's stack */ + int yysize; /* Amount to pop the stack */ + ParseARG_FETCH + (void)yyLookahead; + (void)yyLookaheadToken; + yymsp = yypParser->yytos; + + switch( yyruleno ){ + /* Beginning here are the reduction cases. A typical example + ** follows: + ** case 0: + ** #line <lineno> <grammarfile> + ** { ... } // User supplied code + ** #line <lineno> <thisfile> + ** break; + */ +/********** Begin reduce actions **********************************************/ +%% +/********** End reduce actions ************************************************/ + }; + assert( yyruleno<sizeof(yyRuleInfoLhs)/sizeof(yyRuleInfoLhs[0]) ); + yygoto = yyRuleInfoLhs[yyruleno]; + yysize = yyRuleInfoNRhs[yyruleno]; + yyact = yy_find_reduce_action(yymsp[yysize].stateno,(YYCODETYPE)yygoto); + + /* There are no SHIFTREDUCE actions on nonterminals because the table + ** generator has simplified them to pure REDUCE actions. */ + assert( !(yyact>YY_MAX_SHIFT && yyact<=YY_MAX_SHIFTREDUCE) ); + + /* It is not possible for a REDUCE to be followed by an error */ + assert( yyact!=YY_ERROR_ACTION ); + + yymsp += yysize+1; + yypParser->yytos = yymsp; + yymsp->stateno = (YYACTIONTYPE)yyact; + yymsp->major = (YYCODETYPE)yygoto; + yyTraceShift(yypParser, yyact, "... then shift"); + return yyact; +} + +/* +** The following code executes when the parse fails +*/ +#ifndef YYNOERRORRECOVERY +static void yy_parse_failed( + yyParser *yypParser /* The parser */ +){ + ParseARG_FETCH + ParseCTX_FETCH +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt); + } +#endif + while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); + /* Here code is inserted which will be executed whenever the + ** parser fails */ +/************ Begin %parse_failure code ***************************************/ +%% +/************ End %parse_failure code *****************************************/ + ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ + ParseCTX_STORE +} +#endif /* YYNOERRORRECOVERY */ + +/* +** The following code executes when a syntax error first occurs. +*/ +static void yy_syntax_error( + yyParser *yypParser, /* The parser */ + int yymajor, /* The major type of the error token */ + ParseTOKENTYPE yyminor /* The minor type of the error token */ +){ + ParseARG_FETCH + ParseCTX_FETCH +#define TOKEN yyminor +/************ Begin %syntax_error code ****************************************/ +%% +/************ End %syntax_error code ******************************************/ + ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ + ParseCTX_STORE +} + +/* +** The following is executed when the parser accepts +*/ +static void yy_accept( + yyParser *yypParser /* The parser */ +){ + ParseARG_FETCH + ParseCTX_FETCH +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt); + } +#endif +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + assert( yypParser->yytos==yypParser->yystack ); + /* Here code is inserted which will be executed whenever the + ** parser accepts */ +/*********** Begin %parse_accept code *****************************************/ +%% +/*********** End %parse_accept code *******************************************/ + ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ + ParseCTX_STORE +} + +/* The main parser program. +** The first argument is a pointer to a structure obtained from +** "ParseAlloc" which describes the current state of the parser. +** The second argument is the major token number. The third is +** the minor token. The fourth optional argument is whatever the +** user wants (and specified in the grammar) and is available for +** use by the action routines. +** +** Inputs: +** <ul> +** <li> A pointer to the parser (an opaque structure.) +** <li> The major token number. +** <li> The minor token number. +** <li> An option argument of a grammar-specified type. +** </ul> +** +** Outputs: +** None. +*/ +void Parse( + void *yyp, /* The parser */ + int yymajor, /* The major token code number */ + ParseTOKENTYPE yyminor /* The value for the token */ + ParseARG_PDECL /* Optional %extra_argument parameter */ +){ + YYMINORTYPE yyminorunion; + YYACTIONTYPE yyact; /* The parser action. */ +#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) + int yyendofinput; /* True if we are at the end of input */ +#endif +#ifdef YYERRORSYMBOL + int yyerrorhit = 0; /* True if yymajor has invoked an error */ +#endif + yyParser *yypParser = (yyParser*)yyp; /* The parser */ + ParseCTX_FETCH + ParseARG_STORE + + assert( yypParser->yytos!=0 ); +#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) + yyendofinput = (yymajor==0); +#endif + + yyact = yypParser->yytos->stateno; +#ifndef NDEBUG + if( yyTraceFILE ){ + if( yyact < YY_MIN_REDUCE ){ + fprintf(yyTraceFILE,"%sInput '%s' in state %d\n", + yyTracePrompt,yyTokenName[yymajor],yyact); + }else{ + fprintf(yyTraceFILE,"%sInput '%s' with pending reduce %d\n", + yyTracePrompt,yyTokenName[yymajor],yyact-YY_MIN_REDUCE); + } + } +#endif + + while(1){ /* Exit by "break" */ + assert( yypParser->yytos>=yypParser->yystack ); + assert( yyact==yypParser->yytos->stateno ); + yyact = yy_find_shift_action((YYCODETYPE)yymajor,yyact); + if( yyact >= YY_MIN_REDUCE ){ + unsigned int yyruleno = yyact - YY_MIN_REDUCE; /* Reduce by this rule */ +#ifndef NDEBUG + assert( yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ); + if( yyTraceFILE ){ + int yysize = yyRuleInfoNRhs[yyruleno]; + if( yysize ){ + fprintf(yyTraceFILE, "%sReduce %d [%s]%s, pop back to state %d.\n", + yyTracePrompt, + yyruleno, yyRuleName[yyruleno], + yyruleno<YYNRULE_WITH_ACTION ? "" : " without external action", + yypParser->yytos[yysize].stateno); + }else{ + fprintf(yyTraceFILE, "%sReduce %d [%s]%s.\n", + yyTracePrompt, yyruleno, yyRuleName[yyruleno], + yyruleno<YYNRULE_WITH_ACTION ? "" : " without external action"); + } + } +#endif /* NDEBUG */ + + /* Check that the stack is large enough to grow by a single entry + ** if the RHS of the rule is empty. This ensures that there is room + ** enough on the stack to push the LHS value */ + if( yyRuleInfoNRhs[yyruleno]==0 ){ +#ifdef YYTRACKMAXSTACKDEPTH + if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ + yypParser->yyhwm++; + assert( yypParser->yyhwm == + (int)(yypParser->yytos - yypParser->yystack)); + } +#endif +#if YYSTACKDEPTH>0 + if( yypParser->yytos>=yypParser->yystackEnd ){ + yyStackOverflow(yypParser); + break; + } +#else + if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){ + if( yyGrowStack(yypParser) ){ + yyStackOverflow(yypParser); + break; + } + } +#endif + } + yyact = yy_reduce(yypParser,yyruleno,yymajor,yyminor ParseCTX_PARAM); + }else if( yyact <= YY_MAX_SHIFTREDUCE ){ + yy_shift(yypParser,yyact,(YYCODETYPE)yymajor,yyminor); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt--; +#endif + break; + }else if( yyact==YY_ACCEPT_ACTION ){ + yypParser->yytos--; + yy_accept(yypParser); + return; + }else{ + assert( yyact == YY_ERROR_ACTION ); + yyminorunion.yy0 = yyminor; +#ifdef YYERRORSYMBOL + int yymx; +#endif +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sSyntax Error!\n",yyTracePrompt); + } +#endif +#ifdef YYERRORSYMBOL + /* A syntax error has occurred. + ** The response to an error depends upon whether or not the + ** grammar defines an error token "ERROR". + ** + ** This is what we do if the grammar does define ERROR: + ** + ** * Call the %syntax_error function. + ** + ** * Begin popping the stack until we enter a state where + ** it is legal to shift the error symbol, then shift + ** the error symbol. + ** + ** * Set the error count to three. + ** + ** * Begin accepting and shifting new tokens. No new error + ** processing will occur until three tokens have been + ** shifted successfully. + ** + */ + if( yypParser->yyerrcnt<0 ){ + yy_syntax_error(yypParser,yymajor,yyminor); + } + yymx = yypParser->yytos->major; + if( yymx==YYERRORSYMBOL || yyerrorhit ){ +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sDiscard input token %s\n", + yyTracePrompt,yyTokenName[yymajor]); + } +#endif + yy_destructor(yypParser, (YYCODETYPE)yymajor, &yyminorunion); + yymajor = YYNOCODE; + }else{ + while( yypParser->yytos > yypParser->yystack ){ + yyact = yy_find_reduce_action(yypParser->yytos->stateno, + YYERRORSYMBOL); + if( yyact<=YY_MAX_SHIFTREDUCE ) break; + yy_pop_parser_stack(yypParser); + } + if( yypParser->yytos <= yypParser->yystack || yymajor==0 ){ + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + yy_parse_failed(yypParser); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + yymajor = YYNOCODE; + }else if( yymx!=YYERRORSYMBOL ){ + yy_shift(yypParser,yyact,YYERRORSYMBOL,yyminor); + } + } + yypParser->yyerrcnt = 3; + yyerrorhit = 1; + if( yymajor==YYNOCODE ) break; + yyact = yypParser->yytos->stateno; +#elif defined(YYNOERRORRECOVERY) + /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to + ** do any kind of error recovery. Instead, simply invoke the syntax + ** error routine and continue going as if nothing had happened. + ** + ** Applications can set this macro (for example inside %include) if + ** they intend to abandon the parse upon the first syntax error seen. + */ + yy_syntax_error(yypParser,yymajor, yyminor); + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + break; +#else /* YYERRORSYMBOL is not defined */ + /* This is what we do if the grammar does not define ERROR: + ** + ** * Report an error message, and throw away the input token. + ** + ** * If the input token is $, then fail the parse. + ** + ** As before, subsequent error messages are suppressed until + ** three input tokens have been successfully shifted. + */ + if( yypParser->yyerrcnt<=0 ){ + yy_syntax_error(yypParser,yymajor, yyminor); + } + yypParser->yyerrcnt = 3; + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + if( yyendofinput ){ + yy_parse_failed(yypParser); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + } + break; +#endif + } + } +#ifndef NDEBUG + if( yyTraceFILE ){ + yyStackEntry *i; + char cDiv = '['; + fprintf(yyTraceFILE,"%sReturn. Stack=",yyTracePrompt); + for(i=&yypParser->yystack[1]; i<=yypParser->yytos; i++){ + fprintf(yyTraceFILE,"%c%s", cDiv, yyTokenName[i->major]); + cDiv = ' '; + } + fprintf(yyTraceFILE,"]\n"); + } +#endif + return; +} + +/* +** Return the fallback token corresponding to canonical token iToken, or +** 0 if iToken has no fallback. +*/ +int ParseFallback(int iToken){ +#ifdef YYFALLBACK + assert( iToken<(int)(sizeof(yyFallback)/sizeof(yyFallback[0])) ); + return yyFallback[iToken]; +#else + (void)iToken; + return 0; +#endif +} diff --git a/tool/libvers.c b/tool/libvers.c new file mode 100644 index 0000000..6911dbd --- /dev/null +++ b/tool/libvers.c @@ -0,0 +1,15 @@ +/* +** Compile this program against an SQLite library of unknown version +** and then run this program, and it will print out the SQLite version +** information. +*/ +#include <stdio.h> + +extern const char *sqlite3_libversion(void); +extern const char *sqlite3_sourceid(void); + +int main(int argc, char **argv){ + printf("SQLite version %s\n", sqlite3_libversion()); + printf("SQLite source %s\n", sqlite3_sourceid()); + return 0; +} diff --git a/tool/loadfts.c b/tool/loadfts.c new file mode 100644 index 0000000..0000797 --- /dev/null +++ b/tool/loadfts.c @@ -0,0 +1,242 @@ +/* +** 2014-07-28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements a utility program that will load many disk +** files (all files under a given directory) into a FTS table. This is +** used for performance testing of FTS3, FTS4, and FTS5. +*/ + +#include <stdio.h> +#include <stdlib.h> +#include <ctype.h> +#include <assert.h> +#include <string.h> +#include <errno.h> +#include <dirent.h> +#include "sqlite3.h" + +/* +** Implementation of the "readtext(X)" SQL function. The entire content +** of the file named X is read and returned as a TEXT value. It is assumed +** the file contains UTF-8 text. NULL is returned if the file does not +** exist or is unreadable. +*/ +static void readfileFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zName; + FILE *in; + long nIn; + void *pBuf; + + zName = (const char*)sqlite3_value_text(argv[0]); + if( zName==0 ) return; + in = fopen(zName, "rb"); + if( in==0 ) return; + fseek(in, 0, SEEK_END); + nIn = ftell(in); + rewind(in); + pBuf = sqlite3_malloc( nIn ); + if( pBuf && 1==fread(pBuf, nIn, 1, in) ){ + sqlite3_result_text(context, pBuf, nIn, sqlite3_free); + }else{ + sqlite3_free(pBuf); + } + fclose(in); +} + +/* +** Print usage text for this program and exit. +*/ +static void showHelp(const char *zArgv0){ + printf("\n" +"Usage: %s SWITCHES... DB\n" +"\n" +" This program opens the database named on the command line and attempts to\n" +" create an FTS table named \"fts\" with a single column. If successful, it\n" +" recursively traverses the directory named by the -dir option and inserts\n" +" the contents of each file into the fts table. All files are assumed to\n" +" contain UTF-8 text.\n" +"\n" +"Switches are:\n" +" -fts [345] FTS version to use (default=5)\n" +" -idx [01] Create a mapping from filename to rowid (default=0)\n" +" -dir <path> Root of directory tree to load data from (default=.)\n" +" -trans <integer> Number of inserts per transaction (default=1)\n" +, zArgv0 +); + exit(1); +} + +/* +** Exit with a message based on the argument and the current value of errno. +*/ +static void error_out(const char *zText){ + fprintf(stderr, "%s: %s\n", zText, strerror(errno)); + exit(-1); +} + +/* +** Exit with a message based on the first argument and the error message +** currently stored in database handle db. +*/ +static void sqlite_error_out(const char *zText, sqlite3 *db){ + fprintf(stderr, "%s: %s\n", zText, sqlite3_errmsg(db)); + exit(-1); +} + +/* +** Context object for visit_file(). +*/ +typedef struct VisitContext VisitContext; +struct VisitContext { + int nRowPerTrans; + sqlite3 *db; /* Database handle */ + sqlite3_stmt *pInsert; /* INSERT INTO fts VALUES(readtext(:1)) */ +}; + +/* +** Callback used with traverse(). The first argument points to an object +** of type VisitContext. This function inserts the contents of the text +** file zPath into the FTS table. +*/ +void visit_file(void *pCtx, const char *zPath){ + int rc; + VisitContext *p = (VisitContext*)pCtx; + /* printf("%s\n", zPath); */ + sqlite3_bind_text(p->pInsert, 1, zPath, -1, SQLITE_STATIC); + sqlite3_step(p->pInsert); + rc = sqlite3_reset(p->pInsert); + if( rc!=SQLITE_OK ){ + sqlite_error_out("insert", p->db); + }else if( p->nRowPerTrans>0 + && (sqlite3_last_insert_rowid(p->db) % p->nRowPerTrans)==0 + ){ + sqlite3_exec(p->db, "COMMIT ; BEGIN", 0, 0, 0); + } +} + +/* +** Recursively traverse directory zDir. For each file that is not a +** directory, invoke the supplied callback with its path. +*/ +static void traverse( + const char *zDir, /* Directory to traverse */ + void *pCtx, /* First argument passed to callback */ + void (*xCallback)(void*, const char *zPath) +){ + DIR *d; + struct dirent *e; + + d = opendir(zDir); + if( d==0 ) error_out("opendir()"); + + for(e=readdir(d); e; e=readdir(d)){ + if( strcmp(e->d_name, ".")==0 || strcmp(e->d_name, "..")==0 ) continue; + char *zPath = sqlite3_mprintf("%s/%s", zDir, e->d_name); + if (e->d_type & DT_DIR) { + traverse(zPath, pCtx, xCallback); + }else{ + xCallback(pCtx, zPath); + } + sqlite3_free(zPath); + } + + closedir(d); +} + +int main(int argc, char **argv){ + int iFts = 5; /* Value of -fts option */ + int bMap = 0; /* True to create mapping table */ + const char *zDir = "."; /* Directory to scan */ + int i; + int rc; + int nRowPerTrans = 0; + sqlite3 *db; + char *zSql; + VisitContext sCtx; + + int nCmd = 0; + char **aCmd = 0; + + if( argc % 2 ) showHelp(argv[0]); + + for(i=1; i<(argc-1); i+=2){ + char *zOpt = argv[i]; + char *zArg = argv[i+1]; + if( strcmp(zOpt, "-fts")==0 ){ + iFts = atoi(zArg); + if( iFts!=3 && iFts!=4 && iFts!= 5) showHelp(argv[0]); + } + else if( strcmp(zOpt, "-trans")==0 ){ + nRowPerTrans = atoi(zArg); + } + else if( strcmp(zOpt, "-idx")==0 ){ + bMap = atoi(zArg); + if( bMap!=0 && bMap!=1 ) showHelp(argv[0]); + } + else if( strcmp(zOpt, "-dir")==0 ){ + zDir = zArg; + } + else if( strcmp(zOpt, "-special")==0 ){ + nCmd++; + aCmd = sqlite3_realloc(aCmd, sizeof(char*) * nCmd); + aCmd[nCmd-1] = zArg; + } + else{ + showHelp(argv[0]); + } + } + + /* Open the database file */ + rc = sqlite3_open(argv[argc-1], &db); + if( rc!=SQLITE_OK ) sqlite_error_out("sqlite3_open()", db); + + rc = sqlite3_create_function(db, "readtext", 1, SQLITE_UTF8, 0, + readfileFunc, 0, 0); + if( rc!=SQLITE_OK ) sqlite_error_out("sqlite3_create_function()", db); + + /* Create the FTS table */ + zSql = sqlite3_mprintf("CREATE VIRTUAL TABLE fts USING fts%d(content)", iFts); + rc = sqlite3_exec(db, zSql, 0, 0, 0); + if( rc!=SQLITE_OK ) sqlite_error_out("sqlite3_exec(1)", db); + sqlite3_free(zSql); + + for(i=0; i<nCmd; i++){ + zSql = sqlite3_mprintf("INSERT INTO fts(fts) VALUES(%Q)", aCmd[i]); + rc = sqlite3_exec(db, zSql, 0, 0, 0); + if( rc!=SQLITE_OK ) sqlite_error_out("sqlite3_exec(1)", db); + sqlite3_free(zSql); + } + + /* Compile the INSERT statement to write data to the FTS table. */ + memset(&sCtx, 0, sizeof(VisitContext)); + sCtx.db = db; + sCtx.nRowPerTrans = nRowPerTrans; + rc = sqlite3_prepare_v2(db, + "INSERT INTO fts VALUES(readtext(?))", -1, &sCtx.pInsert, 0 + ); + if( rc!=SQLITE_OK ) sqlite_error_out("sqlite3_prepare_v2(1)", db); + + /* Load all files in the directory hierarchy into the FTS table. */ + if( sCtx.nRowPerTrans>0 ) sqlite3_exec(db, "BEGIN", 0, 0, 0); + traverse(zDir, (void*)&sCtx, visit_file); + if( sCtx.nRowPerTrans>0 ) sqlite3_exec(db, "COMMIT", 0, 0, 0); + + /* Clean up and exit. */ + sqlite3_finalize(sCtx.pInsert); + sqlite3_close(db); + sqlite3_free(aCmd); + return 0; +} diff --git a/tool/logest.c b/tool/logest.c new file mode 100644 index 0000000..d916b43 --- /dev/null +++ b/tool/logest.c @@ -0,0 +1,173 @@ +/* +** 2013-06-10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains a simple command-line utility for converting from +** integers and LogEst values and back again and for doing simple +** arithmetic operations (multiple and add) on LogEst values. +** +** Usage: +** +** ./LogEst ARGS +** +** See the showHelp() routine for a description of valid arguments. +** Examples: +** +** To convert 123 from LogEst to integer: +** +** ./LogEst ^123 +** +** To convert 123456 from integer to LogEst: +** +** ./LogEst 123456 +** +*/ +#include <stdio.h> +#include <stdlib.h> +#include <ctype.h> +#include <assert.h> +#include <string.h> +#include "sqlite3.h" + +typedef short int LogEst; /* 10 times log2() */ + +LogEst logEstMultiply(LogEst a, LogEst b){ return a+b; } +LogEst logEstAdd(LogEst a, LogEst b){ + static const unsigned char x[] = { + 10, 10, /* 0,1 */ + 9, 9, /* 2,3 */ + 8, 8, /* 4,5 */ + 7, 7, 7, /* 6,7,8 */ + 6, 6, 6, /* 9,10,11 */ + 5, 5, 5, /* 12-14 */ + 4, 4, 4, 4, /* 15-18 */ + 3, 3, 3, 3, 3, 3, /* 19-24 */ + 2, 2, 2, 2, 2, 2, 2, /* 25-31 */ + }; + if( a<b ){ LogEst t = a; a = b; b = t; } + if( a>b+49 ) return a; + if( a>b+31 ) return a+1; + return a+x[a-b]; +} +LogEst logEstFromInteger(sqlite3_uint64 x){ + static LogEst a[] = { 0, 2, 3, 5, 6, 7, 8, 9 }; + LogEst y = 40; + if( x<8 ){ + if( x<2 ) return 0; + while( x<8 ){ y -= 10; x <<= 1; } + }else{ + while( x>255 ){ y += 40; x >>= 4; } + while( x>15 ){ y += 10; x >>= 1; } + } + return a[x&7] + y - 10; +} +static sqlite3_uint64 logEstToInt(LogEst x){ + sqlite3_uint64 n; + if( x<10 ) return 1; + n = x%10; + x /= 10; + if( n>=5 ) n -= 2; + else if( n>=1 ) n -= 1; + if( x>60 ) return (((sqlite3_uint64)0xffffffff)<<32)+(sqlite3_uint64)0xffffffff; + if( x>=3 ) return (n+8)<<(x-3); + return (n+8)>>(3-x); +} +static LogEst logEstFromDouble(double x){ + sqlite3_uint64 a; + LogEst e; + assert( sizeof(x)==8 && sizeof(a)==8 ); + if( x<=0.0 ) return -32768; + if( x<0.01 ) return -logEstFromDouble(1.0/x); + if( x<1.0 ) return logEstFromDouble(100.0*x) - 66; + if( x<1024.0 ) return logEstFromInteger((sqlite3_uint64)(1024.0*x)) - 100; + if( x<=2000000000.0 ) return logEstFromInteger((sqlite3_uint64)x); + memcpy(&a, &x, 8); + e = (a>>52) - 1022; + return e*10; +} + +int isInteger(const char *z){ + while( z[0]>='0' && z[0]<='9' ) z++; + return z[0]==0; +} + +int isFloat(const char *z){ + char c; + while( ((c=z[0])>='0' && c<='9') || c=='.' || c=='E' || c=='e' + || c=='+' || c=='-' ) z++; + return z[0]==0; +} + +static void showHelp(const char *zArgv0){ + printf("Usage: %s ARGS...\n", zArgv0); + printf("Arguments:\n" + " NUM Convert NUM from integer to LogEst and push onto the stack\n" + " ^NUM Interpret NUM as a LogEst and push onto stack\n" + " x Multiple the top two elements of the stack\n" + " + Add the top two elements of the stack\n" + " dup Dupliate the top element on the stack\n" + " inv Take the reciprocal of the top of stack. N = 1/N.\n" + " log Find the LogEst of the number on top of stack\n" + " nlogn Compute NlogN where N is the top of stack\n" + ); + exit(1); +} + +int main(int argc, char **argv){ + int i; + int n = 0; + LogEst a[100]; + for(i=1; i<argc; i++){ + const char *z = argv[i]; + if( strcmp(z,"+")==0 ){ + if( n>=2 ){ + a[n-2] = logEstAdd(a[n-2],a[n-1]); + n--; + } + }else if( strcmp(z,"x")==0 ){ + if( n>=2 ){ + a[n-2] = logEstMultiply(a[n-2],a[n-1]); + n--; + } + }else if( strcmp(z,"dup")==0 ){ + if( n>0 ){ + a[n] = a[n-1]; + n++; + } + }else if( strcmp(z,"log")==0 ){ + if( n>0 ) a[n-1] = logEstFromInteger(a[n-1]) - 33; + }else if( strcmp(z,"nlogn")==0 ){ + if( n>0 ) a[n-1] += logEstFromInteger(a[n-1]) - 33; + }else if( strcmp(z,"inv")==0 ){ + if( n>0 ) a[n-1] = -a[n-1]; + }else if( z[0]=='^' ){ + a[n++] = (LogEst)atoi(z+1); + }else if( isInteger(z) ){ + a[n++] = logEstFromInteger(atoll(z)); + }else if( isFloat(z) && z[0]!='-' ){ + a[n++] = logEstFromDouble(atof(z)); + }else{ + showHelp(argv[0]); + } + } + for(i=n-1; i>=0; i--){ + if( a[i]<-40 ){ + printf("%5d (%f)\n", a[i], 1.0/(double)logEstToInt(-a[i])); + }else if( a[i]<10 ){ + printf("%5d (%f)\n", a[i], logEstToInt(a[i]+100)/1024.0); + }else if( a[i]>100 ){ + printf("%5d (%lld)\n", a[i], logEstToInt(a[i])); + }else{ + sqlite3_uint64 x = logEstToInt(a[i]+100)*100/1024; + printf("%5d (%lld.%02lld)\n", a[i], x/100, x%100); + } + } + return 0; +} diff --git a/tool/max-limits.c b/tool/max-limits.c new file mode 100644 index 0000000..d019974 --- /dev/null +++ b/tool/max-limits.c @@ -0,0 +1,41 @@ +/* +** Link this program against an SQLite library of unknown provenance in order +** to display the compile-time maximum values for various settings. +*/ +#include "sqlite3.h" +#include <stdio.h> + +static const struct { + int eCode; + char *zName; +} aLimit[] = { + { SQLITE_LIMIT_LENGTH, "SQLITE_MAX_LENGTH" }, + { SQLITE_LIMIT_SQL_LENGTH, "SQLITE_MAX_SQL_LENGTH" }, + { SQLITE_LIMIT_COLUMN, "SQLITE_MAX_COLUMN" }, + { SQLITE_LIMIT_EXPR_DEPTH, "SQLITE_MAX_EXPR_DEPTH" }, + { SQLITE_LIMIT_COMPOUND_SELECT, "SQLITE_MAX_COMPOUND_SELECT" }, + { SQLITE_LIMIT_VDBE_OP, "SQLITE_MAX_VDBE_OP" }, + { SQLITE_LIMIT_FUNCTION_ARG, "SQLITE_MAX_FUNCTION_ARG" }, + { SQLITE_LIMIT_ATTACHED, "SQLITE_MAX_ATTACHED" }, + { SQLITE_LIMIT_LIKE_PATTERN_LENGTH, "SQLITE_MAX_LIKE_PATTERN_LENGTH" }, + { SQLITE_LIMIT_VARIABLE_NUMBER, "SQLITE_MAX_VARIABLE_NUMBER" }, + { SQLITE_LIMIT_TRIGGER_DEPTH, "SQLITE_MAX_TRIGGER_DEPTH" }, + { SQLITE_LIMIT_WORKER_THREADS, "SQLITE_MAX_WORKER_THREADS" }, +}; + +static int maxLimit(sqlite3 *db, int eCode){ + int iOrig = sqlite3_limit(db, eCode, 0x7fffffff); + return sqlite3_limit(db, eCode, iOrig); +} + +int main(int argc, char **argv){ + sqlite3 *db; + int j, rc; + rc = sqlite3_open(":memory:", &db); + if( rc==SQLITE_OK ){ + for(j=0; j<sizeof(aLimit)/sizeof(aLimit[0]); j++){ + printf("%-35s %10d\n", aLimit[j].zName, maxLimit(db, aLimit[j].eCode)); + } + sqlite3_close(db); + } +} diff --git a/tool/merge-test.tcl b/tool/merge-test.tcl new file mode 100644 index 0000000..2010d67 --- /dev/null +++ b/tool/merge-test.tcl @@ -0,0 +1,99 @@ +#!/usr/bin/tcl +# +# Run this script to test to see that the latest trunk changes can be +# merged into LTS branches without breaking anything. +# +# To Use: +# +# * Copy this script into a directory above the sqlite checkout +# * Run "fossil update trunk" and "fossil revert" +# * Run "tclsh ../merge-test.tcl" (in other words run this script) +# +# Operation: +# +# This script changes to each LTS branch to be tested, merges the latest +# trunk changes into the branch (without committing them) and then +# runs "make test". Any errors are stored in local files. +# +# Limitations: +# +# Some LTS branches are not synced directly from trunk but rather from +# other LTS branches. These other branches cannot be tested because +# there is no good way to generate the intermediate merges. +# +############################################################################### + +# Run a shell command contained in arguments. Put the return code in +# global variable ::res and the output string in global variable ::result +# +proc safeexec {args} { + global res result + set res [catch "exec $args" result] +} + +# Run the shell command contained in arguments. Print an error and exit +# if anything goes wrong. +# +proc mustbeok {args} { + global res result + set res [catch "exec $args" result] + if {$res} { + puts "FAILED: $args" + puts $result + exit 1 + } +} + +# Write $content into a file named $filename. The file is overwritten if it +# already exist. The file is create if it does not already exist. +# +proc writefile {filename content} { + set fd [open $filename wb] + puts $fd $content + close $fd +} + +# Run the merge-test +# +foreach {branch configopts} { + begin-concurrent {--enable-json1} + begin-concurrent-pnu {--enable-json1} + wal2 {--enable-all} + reuse-schema {--enable-all} +} { + puts $branch + set errorfile ${branch}-error.txt + mustbeok fossil revert + mustbeok fossil up $branch + safeexec fossil merge trunk + if {$res} { + puts " merge failed - see $errorfile" + writefile $errorfile $result + } else { + puts " merge ok" + safeexec ./configure --enable-debug {*}$configopts + if {$res} { + puts " configure failed - see $errorfile" + writefile $errorfile $result + } else { + puts " configure ok" + safeexec make fuzzcheck sqlite3 testfixture + if {$res} { + puts " build failed - see $errorfile" + writefile $errorfile $result + } else { + puts " build ok" + safeexec make test + if {$res} { + puts " test failed - see $errorfile" + writefile $errorfile $result + } else { + puts " test ok" + } + } + } + } +} +mustbeok fossil revert +mustbeok fossil up trunk +puts "reset back to trunk" diff --git a/tool/mkautoconfamal.sh b/tool/mkautoconfamal.sh new file mode 100644 index 0000000..35dbfb4 --- /dev/null +++ b/tool/mkautoconfamal.sh @@ -0,0 +1,101 @@ +#!/bin/sh +# This script is used to build the amalgamation autoconf package. +# It assumes the following: +# +# 1. The files "sqlite3.c", "sqlite3.h", "sqlite3ext.h", "shell.c", +# and "sqlite3rc.h" are available in the current directory. +# +# 2. Variable $TOP is set to the full path of the root directory +# of the SQLite source tree. +# +# 3. There is nothing of value in the ./mkpkg_tmp_dir directory. +# This is important, as the script executes "rm -rf ./mkpkg_tmp_dir". +# + + +# Bail out of the script if any command returns a non-zero exit +# status. Or if the script tries to use an unset variable. These +# may fail for old /bin/sh interpreters. +# +set -e +set -u + +TMPSPACE=./mkpkg_tmp_dir +VERSION=`cat $TOP/VERSION` +HASH=`sed 's/^\(..........\).*/\1/' $TOP/manifest.uuid` +DATETIME=`grep '^D' $TOP/manifest | sed -e 's/[^0-9]//g' -e 's/\(............\).*/\1/'` + +# Verify that the version number in the TEA autoconf file is correct. +# Fail with an error if not. +# +if grep $VERSION $TOP/autoconf/tea/configure.ac +then echo "TEA version number ok" +else echo "TEA version number mismatch. Should be $VERSION"; exit 1 +fi + +# If this script is given an argument of --snapshot, then generate a +# snapshot tarball named for the current checkout SHA1 hash, rather than +# the version number. +# +if test "$#" -ge 1 -a x$1 != x--snapshot +then + # Set global variable $ARTIFACT to the "3xxyyzz" string incorporated + # into artifact filenames. And $VERSION2 to the "3.x.y[.z]" form. + xx=`echo $VERSION|sed 's/3\.\([0-9]*\)\..*/\1/'` + yy=`echo $VERSION|sed 's/3\.[^.]*\.\([0-9]*\).*/\1/'` + zz=0 + set +e + zz=`echo $VERSION|sed 's/3\.[^.]*\.[^.]*\.\([0-9]*\).*/\1/'|grep -v '\.'` + set -e + TARBALLNAME=`printf "sqlite-autoconf-3%.2d%.2d%.2d" $xx $yy $zz` +else + TARBALLNAME=sqlite-snapshot-$DATETIME +fi + +rm -rf $TMPSPACE +cp -R $TOP/autoconf $TMPSPACE +cp sqlite3.c $TMPSPACE +cp sqlite3.h $TMPSPACE +cp sqlite3ext.h $TMPSPACE +cp sqlite3rc.h $TMPSPACE +cp $TOP/sqlite3.1 $TMPSPACE +cp $TOP/sqlite3.pc.in $TMPSPACE +cp shell.c $TMPSPACE +cp $TOP/src/sqlite3.rc $TMPSPACE +cp $TOP/tool/Replace.cs $TMPSPACE + +cat $TMPSPACE/configure.ac | +sed "s/--SQLITE-VERSION--/$VERSION/" > $TMPSPACE/tmp +mv $TMPSPACE/tmp $TMPSPACE/configure.ac + +cd $TMPSPACE +autoreconf -i +#libtoolize +#aclocal +#autoconf +#automake --add-missing + +mkdir -p tea/generic +echo "#ifdef USE_SYSTEM_SQLITE" > tea/generic/tclsqlite3.c +echo "# include <sqlite3.h>" >> tea/generic/tclsqlite3.c +echo "#else" >> tea/generic/tclsqlite3.c +echo "#include \"sqlite3.c\"" >> tea/generic/tclsqlite3.c +echo "#endif" >> tea/generic/tclsqlite3.c +cat $TOP/src/tclsqlite.c >> tea/generic/tclsqlite3.c + +cat tea/configure.ac | + sed "s/AC_INIT(\[sqlite\], .*)/AC_INIT([sqlite], [$VERSION])/" > tmp +mv tmp tea/configure.ac + +cd tea +autoconf +rm -rf autom4te.cache + +cd ../ +./configure && make dist +tar -xzf sqlite-$VERSION.tar.gz +mv sqlite-$VERSION $TARBALLNAME +tar -czf $TARBALLNAME.tar.gz $TARBALLNAME +mv $TARBALLNAME.tar.gz .. +cd .. +ls -l $TARBALLNAME.tar.gz diff --git a/tool/mkccode.tcl b/tool/mkccode.tcl new file mode 100755 index 0000000..41b09f1 --- /dev/null +++ b/tool/mkccode.tcl @@ -0,0 +1,93 @@ +#!/usr/bin/tclsh +# +# Use this script to build C-language source code for a program that uses +# tclsqlite.c together with custom TCL scripts and/or C extensions for +# either SQLite or TCL. +# +# Usage example: +# +# tclsh mktclsqliteprog.tcl demoapp.c.in >demoapp.c +# +# The demoapp.c.in file contains a mixture of C code, TCL script, and +# processing directives used by mktclsqliteprog.tcl to build the final C-code +# output file. Most lines of demoapp.c.in are copied straight through into +# the output. The following control directives are recognized: +# +# BEGIN_STRING +# +# This marks the beginning of large string literal - usually a TCL +# script of some kind. Subsequent lines of text through the first +# line that begins with END_STRING are converted into a C-language +# string literal. +# +# INCLUDE path +# +# The path argument is the name of a file to be inserted in place of +# the INCLUDE line. The path can begin with $ROOT to signify the +# root of the SQLite source tree, or $HOME to signify the directory +# that contains the demoapp.c.in input script itself. If the path does +# not begin with either $ROOT or $HOME, then it is interpreted relative +# to the current working directory. +# +# If the INCLUDE occurs in the middle of BEGIN_STRING...END_STRING +# then all of the text in the input file is converted into C-language +# string literals. +# +# None of the control directives described above will nest. Only the +# top-level input file ("demoapp.c.in" in the example) is interpreted. +# referenced files are copied verbatim. +# +if {[llength $argv]!=1} { + puts stderr "Usage: $argv0 TEMPLATE >OUTPUT" + exit 1 +} +set infile [lindex $argv 0] +set ROOT [file normalize [file dir $argv0]/..] +set HOME [file normalize [file dir $infile]] +set in [open $infile rb] +puts [subst {/* DO NOT EDIT +** +** This file was generated by \"$argv0 $infile\". +** To make changes, edit $infile then rerun the generator +** command. +*/}] +set instr 0 +while {1} { + set line [gets $in] + if {[eof $in]} break + if {[regexp {^INCLUDE (.*)} $line all path]} { + regsub {^\$ROOT\y} $path $ROOT path + regsub {^\$HOME\y} $path $HOME path + set in2 [open $path rb] + puts "/* INCLUDE $path */" + if {$instr} { + while {1} { + set line [gets $in2] + if {[eof $in2]} break + set x [string map "\\\\ \\\\\\\\ \\\" \\\\\"" $line] + puts "\"$x\\n\"" + } + } else { + puts [read $in2] + } + puts "/* END $path */" + close $in2 + continue + } + if {[regexp {^BEGIN_STRING} $line]} { + set instr 1 + puts "/* BEGIN_STRING */" + continue + } + if {[regexp {^END_STRING} $line]} { + set instr 0 + puts "/* END_STRING */" + continue + } + if {$instr} { + set x [string map "\\\\ \\\\\\\\ \\\" \\\\\"" $line] + puts "\"$x\\n\"" + } else { + puts $line + } +} diff --git a/tool/mkctimec.tcl b/tool/mkctimec.tcl new file mode 100755 index 0000000..9e425c0 --- /dev/null +++ b/tool/mkctimec.tcl @@ -0,0 +1,458 @@ +#!/usr/bin/tclsh +# +# To build the +# +# const char **azCompileOpt[] +# +# definition used in src/ctime.c, run this script from +# the checkout root. It generates src/ctime.c . +# +# Results are normally written into src/ctime.c. But if an argument is +# provided, results are written there instead. Examples: +# +# tclsh tool/mkctimec.tcl ;# <-- results to src/ctime.c +# +# tclsh tool/mkctimec.tcl /dev/tty ;# <-- results to the terminal +# + + +set ::headWarning {/* DO NOT EDIT! +** This file is automatically generated by the script in the canonical +** SQLite source tree at tool/mkctimec.tcl. +** +** To modify this header, edit any of the various lists in that script +** which specify categories of generated conditionals in this file. +*/} + +# Make { and } easier to put into literals (even on EBCDIC machines.) +regexp {(\{)(\})} "{}" ma ::lb ::rb + +set ::headCode " +/* +** 2010 February 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements routines used to report what compile-time options +** SQLite was built with. +*/ +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS /* IMP: R-16824-07538 */ + +/* +** Include the configuration header output by 'configure' if we're using the +** autoconf-based build +*/ +#if defined(_HAVE_SQLITE_CONFIG_H) && !defined(SQLITECONFIG_H) +#include \"sqlite_cfg.h\" +#define SQLITECONFIG_H 1 +#endif + +/* These macros are provided to \"stringify\" the value of the define +** for those options in which the value is meaningful. */ +#define CTIMEOPT_VAL_(opt) #opt +#define CTIMEOPT_VAL(opt) CTIMEOPT_VAL_(opt) + +/* Like CTIMEOPT_VAL, but especially for SQLITE_DEFAULT_LOOKASIDE. This +** option requires a separate macro because legal values contain a single +** comma. e.g. (-DSQLITE_DEFAULT_LOOKASIDE=\"100,100\") */ +#define CTIMEOPT_VAL2_(opt1,opt2) #opt1 \",\" #opt2 +#define CTIMEOPT_VAL2(opt) CTIMEOPT_VAL2_(opt) +#include \"sqliteInt.h\" + +/* +** An array of names of all compile-time options. This array should +** be sorted A-Z. +** +** This array looks large, but in a typical installation actually uses +** only a handful of compile-time options, so most times this array is usually +** rather short and uses little memory space. +*/ +static const char * const sqlite3azCompileOpt\[\] = $::lb +" + +set ::tailCode " +$::rb ; + +const char **sqlite3CompileOptions(int *pnOpt){ + *pnOpt = sizeof(sqlite3azCompileOpt) / sizeof(sqlite3azCompileOpt\[0\]); + return (const char**)sqlite3azCompileOpt; +} + +#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ +" + +# All Boolean compile time options which default to something +# other than 0 or empty. The default is paired with the PP +# symbol so that a differing define can be detected. +# +set boolean_defnnz_options { + {SQLITE_HOMEGROWN_RECURSIVE_MUTEX 1} + {SQLITE_POWERSAFE_OVERWRITE 1} + {SQLITE_DEFAULT_MEMSTATUS 1} + {SQLITE_OMIT_TRACE 1} + {SQLITE_ALLOW_COVERING_INDEX_SCAN 1} +} + +# All Boolean compile time options which default to 0 or empty. +# +set boolean_defnil_options { + SQLITE_32BIT_ROWID + SQLITE_4_BYTE_ALIGNED_MALLOC + SQLITE_ALLOW_URI_AUTHORITY + SQLITE_BUG_COMPATIBLE_20160819 + SQLITE_CASE_SENSITIVE_LIKE + SQLITE_CHECK_PAGES + SQLITE_COVERAGE_TEST + SQLITE_DEBUG + SQLITE_DEFAULT_AUTOMATIC_INDEX + SQLITE_DEFAULT_AUTOVACUUM + SQLITE_DEFAULT_CKPTFULLFSYNC + SQLITE_DEFAULT_FOREIGN_KEYS + SQLITE_DEFAULT_LOCKING_MODE + SQLITE_DEFAULT_RECURSIVE_TRIGGERS + SQLITE_DEFAULT_SYNCHRONOUS + SQLITE_DEFAULT_WAL_SYNCHRONOUS + SQLITE_DIRECT_OVERFLOW_READ + SQLITE_DISABLE_DIRSYNC + SQLITE_DISABLE_FTS3_UNICODE + SQLITE_DISABLE_FTS4_DEFERRED + SQLITE_DISABLE_INTRINSIC + SQLITE_DISABLE_LFS + SQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS + SQLITE_DISABLE_SKIPAHEAD_DISTINCT + SQLITE_ENABLE_8_3_NAMES + SQLITE_ENABLE_API_ARMOR + SQLITE_ENABLE_ATOMIC_WRITE + SQLITE_ENABLE_BATCH_ATOMIC_WRITE + SQLITE_ENABLE_BYTECODE_VTAB + SQLITE_ENABLE_COLUMN_METADATA + SQLITE_ENABLE_COLUMN_USED_MASK + SQLITE_ENABLE_COSTMULT + SQLITE_ENABLE_CURSOR_HINTS + SQLITE_ENABLE_DBPAGE_VTAB + SQLITE_ENABLE_DBSTAT_VTAB + SQLITE_ENABLE_EXPENSIVE_ASSERT + SQLITE_ENABLE_EXPLAIN_COMMENTS + SQLITE_ENABLE_FTS3 + SQLITE_ENABLE_FTS3_PARENTHESIS + SQLITE_ENABLE_FTS3_TOKENIZER + SQLITE_ENABLE_FTS4 + SQLITE_ENABLE_FTS5 + SQLITE_ENABLE_GEOPOLY + SQLITE_ENABLE_HIDDEN_COLUMNS + SQLITE_ENABLE_ICU + SQLITE_ENABLE_IOTRACE + SQLITE_ENABLE_LOAD_EXTENSION + SQLITE_ENABLE_LOCKING_STYLE + SQLITE_ENABLE_MATH_FUNCTIONS + SQLITE_ENABLE_MEMORY_MANAGEMENT + SQLITE_ENABLE_MEMSYS3 + SQLITE_ENABLE_MEMSYS5 + SQLITE_ENABLE_MULTIPLEX + SQLITE_ENABLE_NORMALIZE + SQLITE_ENABLE_NULL_TRIM + SQLITE_ENABLE_OFFSET_SQL_FUNC + SQLITE_ENABLE_OVERSIZE_CELL_CHECK + SQLITE_ENABLE_PREUPDATE_HOOK + SQLITE_ENABLE_QPSG + SQLITE_ENABLE_RBU + SQLITE_ENABLE_RTREE + SQLITE_ENABLE_SESSION + SQLITE_ENABLE_SNAPSHOT + SQLITE_ENABLE_SORTER_REFERENCES + SQLITE_ENABLE_SQLLOG + SQLITE_ENABLE_STAT4 + SQLITE_ENABLE_STMT_SCANSTATUS + SQLITE_ENABLE_STMTVTAB + SQLITE_ENABLE_TREETRACE + SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + SQLITE_ENABLE_UNLOCK_NOTIFY + SQLITE_ENABLE_UPDATE_DELETE_LIMIT + SQLITE_ENABLE_URI_00_ERROR + SQLITE_ENABLE_VFSTRACE + SQLITE_ENABLE_WHERETRACE + SQLITE_ENABLE_ZIPVFS + SQLITE_EXPLAIN_ESTIMATED_ROWS + SQLITE_EXTRA_IFNULLROW + SQLITE_FTS5_ENABLE_TEST_MI + SQLITE_FTS5_NO_WITHOUT_ROWID + SQLITE_IGNORE_AFP_LOCK_ERRORS + SQLITE_IGNORE_FLOCK_LOCK_ERRORS + SQLITE_INLINE_MEMCPY + SQLITE_INT64_TYPE + SQLITE_LEGACY_JSON_VALID + SQLITE_LIKE_DOESNT_MATCH_BLOBS + SQLITE_LOCK_TRACE + SQLITE_LOG_CACHE_SPILL + SQLITE_MEMDEBUG + SQLITE_MIXED_ENDIAN_64BIT_FLOAT + SQLITE_MMAP_READWRITE + SQLITE_MUTEX_NOOP + SQLITE_MUTEX_OMIT + SQLITE_MUTEX_PTHREADS + SQLITE_MUTEX_W32 + SQLITE_NEED_ERR_NAME + SQLITE_NO_SYNC + SQLITE_OMIT_ALTERTABLE + SQLITE_OMIT_ANALYZE + SQLITE_OMIT_ATTACH + SQLITE_OMIT_AUTHORIZATION + SQLITE_OMIT_AUTOINCREMENT + SQLITE_OMIT_AUTOINIT + SQLITE_OMIT_AUTOMATIC_INDEX + SQLITE_OMIT_AUTORESET + SQLITE_OMIT_AUTOVACUUM + SQLITE_OMIT_BETWEEN_OPTIMIZATION + SQLITE_OMIT_BLOB_LITERAL + SQLITE_OMIT_CAST + SQLITE_OMIT_CHECK + SQLITE_OMIT_COMPLETE + SQLITE_OMIT_COMPOUND_SELECT + SQLITE_OMIT_CONFLICT_CLAUSE + SQLITE_OMIT_CTE + SQLITE_OMIT_DECLTYPE + SQLITE_OMIT_DEPRECATED + SQLITE_OMIT_DESERIALIZE + SQLITE_OMIT_DISKIO + SQLITE_OMIT_EXPLAIN + SQLITE_OMIT_FLAG_PRAGMAS + SQLITE_OMIT_FLOATING_POINT + SQLITE_OMIT_FOREIGN_KEY + SQLITE_OMIT_GET_TABLE + SQLITE_OMIT_HEX_INTEGER + SQLITE_OMIT_INCRBLOB + SQLITE_OMIT_INTEGRITY_CHECK + SQLITE_OMIT_INTROSPECTION_PRAGMAS + SQLITE_OMIT_JSON + SQLITE_OMIT_LIKE_OPTIMIZATION + SQLITE_OMIT_LOAD_EXTENSION + SQLITE_OMIT_LOCALTIME + SQLITE_OMIT_LOOKASIDE + SQLITE_OMIT_MEMORYDB + SQLITE_OMIT_OR_OPTIMIZATION + SQLITE_OMIT_PAGER_PRAGMAS + SQLITE_OMIT_PARSER_TRACE + SQLITE_OMIT_POPEN + SQLITE_OMIT_PRAGMA + SQLITE_OMIT_PROGRESS_CALLBACK + SQLITE_OMIT_QUICKBALANCE + SQLITE_OMIT_REINDEX + SQLITE_OMIT_SCHEMA_PRAGMAS + SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS + SQLITE_OMIT_SEH + SQLITE_OMIT_SHARED_CACHE + SQLITE_OMIT_SHUTDOWN_DIRECTORIES + SQLITE_OMIT_SUBQUERY + SQLITE_OMIT_TCL_VARIABLE + SQLITE_OMIT_TEMPDB + SQLITE_OMIT_TEST_CONTROL + SQLITE_OMIT_TRIGGER + SQLITE_OMIT_TRUNCATE_OPTIMIZATION + SQLITE_OMIT_UTF16 + SQLITE_OMIT_VACUUM + SQLITE_OMIT_VIEW + SQLITE_OMIT_VIRTUALTABLE + SQLITE_OMIT_WAL + SQLITE_OMIT_WSD + SQLITE_OMIT_XFER_OPT + SQLITE_PERFORMANCE_TRACE + SQLITE_PREFER_PROXY_LOCKING + SQLITE_PROXY_DEBUG + SQLITE_REVERSE_UNORDERED_SELECTS + SQLITE_RTREE_INT_ONLY + SQLITE_SECURE_DELETE + SQLITE_SMALL_STACK + SQLITE_SOUNDEX + SQLITE_SUBSTR_COMPATIBILITY + SQLITE_TCL + SQLITE_TEST + SQLITE_UNLINK_AFTER_CLOSE + SQLITE_UNTESTABLE + SQLITE_USE_ALLOCA + SQLITE_USE_FCNTL_TRACE + SQLITE_USER_AUTHENTICATION + SQLITE_USE_URI + SQLITE_VDBE_COVERAGE + SQLITE_WIN32_MALLOC + SQLITE_ZERO_MALLOC +} + +# All compile time options for which the assigned value is other than boolean +# and is a comma-separated scalar pair. +# +set value2_options { + SQLITE_DEFAULT_LOOKASIDE +} + +# All compile time options for which the assigned value is other than boolean +# and is a single scalar. +# +set value_options { + SQLITE_ATOMIC_INTRINSICS + SQLITE_BITMASK_TYPE + SQLITE_DEFAULT_CACHE_SIZE + SQLITE_DEFAULT_FILE_FORMAT + SQLITE_DEFAULT_FILE_PERMISSIONS + SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT + SQLITE_DEFAULT_LOCKING_MODE + SQLITE_DEFAULT_MMAP_SIZE + SQLITE_DEFAULT_PAGE_SIZE + SQLITE_DEFAULT_PCACHE_INITSZ + SQLITE_DEFAULT_PROXYDIR_PERMISSIONS + SQLITE_DEFAULT_ROWEST + SQLITE_DEFAULT_SECTOR_SIZE + SQLITE_DEFAULT_SYNCHRONOUS + SQLITE_DEFAULT_WAL_AUTOCHECKPOINT + SQLITE_DEFAULT_WAL_SYNCHRONOUS + SQLITE_DEFAULT_WORKER_THREADS + SQLITE_DQS + SQLITE_ENABLE_8_3_NAMES + SQLITE_ENABLE_CEROD + SQLITE_ENABLE_LOCKING_STYLE + SQLITE_EXTRA_AUTOEXT + SQLITE_EXTRA_INIT + SQLITE_EXTRA_SHUTDOWN + SQLITE_FTS3_MAX_EXPR_DEPTH + SQLITE_INTEGRITY_CHECK_ERROR_MAX + SQLITE_MALLOC_SOFT_LIMIT + SQLITE_MAX_ATTACHED + SQLITE_MAX_COLUMN + SQLITE_MAX_COMPOUND_SELECT + SQLITE_MAX_DEFAULT_PAGE_SIZE + SQLITE_MAX_EXPR_DEPTH + SQLITE_MAX_FUNCTION_ARG + SQLITE_MAX_LENGTH + SQLITE_MAX_LIKE_PATTERN_LENGTH + SQLITE_MAX_MEMORY + SQLITE_MAX_MMAP_SIZE + SQLITE_MAX_MMAP_SIZE_ + SQLITE_MAX_PAGE_COUNT + SQLITE_MAX_PAGE_SIZE + SQLITE_MAX_SCHEMA_RETRY + SQLITE_MAX_SQL_LENGTH + SQLITE_MAX_TRIGGER_DEPTH + SQLITE_MAX_VARIABLE_NUMBER + SQLITE_MAX_VDBE_OP + SQLITE_MAX_WORKER_THREADS + SQLITE_SORTER_PMASZ + SQLITE_STAT4_SAMPLES + SQLITE_STMTJRNL_SPILL + SQLITE_TEMP_STORE +} + +# Options that require custom code. +# +set options(COMPILER) { +#if defined(__clang__) && defined(__clang_major__) + "COMPILER=clang-" CTIMEOPT_VAL(__clang_major__) "." + CTIMEOPT_VAL(__clang_minor__) "." + CTIMEOPT_VAL(__clang_patchlevel__), +#elif defined(_MSC_VER) + "COMPILER=msvc-" CTIMEOPT_VAL(_MSC_VER), +#elif defined(__GNUC__) && defined(__VERSION__) + "COMPILER=gcc-" __VERSION__, +#endif +} +set options(HAVE_ISNAN) { +#if HAVE_ISNAN || SQLITE_HAVE_ISNAN + "HAVE_ISNAN", +#endif +} +set options(OMIT_DATETIME_FUNCS) { +#if defined(SQLITE_OMIT_DATETIME_FUNCS) || defined(SQLITE_OMIT_FLOATING_POINT) + "OMIT_DATETIME_FUNCS", +#endif +} +set options(SYSTEM_MALLOC) "\ +#if (!defined(SQLITE_WIN32_MALLOC) \\ + && !defined(SQLITE_ZERO_MALLOC) \\ + && !defined(SQLITE_MEMDEBUG) \\ + ) || defined(SQLITE_SYSTEM_MALLOC) + \"SYSTEM_MALLOC\", +#endif +" +set options(THREADSAFE) { +#if defined(SQLITE_THREADSAFE) + "THREADSAFE=" CTIMEOPT_VAL(SQLITE_THREADSAFE), +#elif defined(THREADSAFE) + "THREADSAFE=" CTIMEOPT_VAL(THREADSAFE), +#else + "THREADSAFE=1", +#endif +} + +proc trim_name {in} { + set ret $in + if {[string range $in 0 6]=="SQLITE_"} { + set ret [string range $in 7 end] + } + return $ret +} + +foreach name_defval $boolean_defnnz_options { + set b [lindex $name_defval 0] + set defval [lindex $name_defval 1] + set name [trim_name $b] + set options($name) [subst { +#ifdef $b +# if $b != $defval + "$name=" CTIMEOPT_VAL($b), +# endif +#endif +}] +} + +foreach b $boolean_defnil_options { + set name [trim_name $b] + set options($name) [subst { +#ifdef $b + "$name", +#endif +}] +} + +foreach v $value_options { + set name [trim_name $v] + set options($name) [subst { +#ifdef $v + "$name=" CTIMEOPT_VAL($v), +#endif +}] +} + +foreach v $value2_options { + set name [trim_name $v] + set options($name) [subst { +#ifdef $v + "$name=" CTIMEOPT_VAL2($v), +#endif +}] +} + +if {$argc>0} { + set destfile [lindex $argv 0] +} else { + set destfile "[file dir [file dir [file normal $argv0]]]/src/ctime.c" + puts "Overwriting $destfile..." +} + +if {[catch {set cfd [open $destfile w]}]!=0} { + puts stderr "File '$destfile' unwritable." + exit 1; +} + +puts $cfd $::headWarning; +puts $cfd $::headCode; +foreach o [lsort [array names options]] { + puts $cfd [string trim $options($o)] +} +puts -nonewline $cfd $::tailCode; + +close $cfd diff --git a/tool/mkkeywordhash.c b/tool/mkkeywordhash.c new file mode 100644 index 0000000..5386a36 --- /dev/null +++ b/tool/mkkeywordhash.c @@ -0,0 +1,716 @@ +/* +** Compile and run this standalone program in order to generate code that +** implements a function that will translate alphabetic identifiers into +** parser token codes. +*/ +#include <stdio.h> +#include <string.h> +#include <stdlib.h> +#include <assert.h> + +/* +** A header comment placed at the beginning of generated code. +*/ +static const char zHdr[] = + "/***** This file contains automatically generated code ******\n" + "**\n" + "** The code in this file has been automatically generated by\n" + "**\n" + "** sqlite/tool/mkkeywordhash.c\n" + "**\n" + "** The code in this file implements a function that determines whether\n" + "** or not a given identifier is really an SQL keyword. The same thing\n" + "** might be implemented more directly using a hand-written hash table.\n" + "** But by using this automatically generated code, the size of the code\n" + "** is substantially reduced. This is important for embedded applications\n" + "** on platforms with limited memory.\n" + "*/\n" +; + +/* +** All the keywords of the SQL language are stored in a hash +** table composed of instances of the following structure. +*/ +typedef struct Keyword Keyword; +struct Keyword { + char *zName; /* The keyword name */ + char *zTokenType; /* Token value for this keyword */ + int mask; /* Code this keyword if non-zero */ + int priority; /* Put higher priorities earlier in the hash chain */ + int id; /* Unique ID for this record */ + int hash; /* Hash on the keyword */ + int offset; /* Offset to start of name string */ + int len; /* Length of this keyword, not counting final \000 */ + int prefix; /* Number of characters in prefix */ + int longestSuffix; /* Longest suffix that is a prefix on another word */ + int iNext; /* Index in aKeywordTable[] of next with same hash */ + int substrId; /* Id to another keyword this keyword is embedded in */ + int substrOffset; /* Offset into substrId for start of this keyword */ + char zOrigName[20]; /* Original keyword name before processing */ +}; + +/* +** Define masks used to determine which keywords are allowed +*/ +#if defined(SQLITE_OMIT_ALTERTABLE) || defined(SQLITE_OMIT_VIRTUALTABLE) +# define ALTER 0 +#else +# define ALTER 0x00000001 +#endif +#define ALWAYS 0x00000002 +#ifdef SQLITE_OMIT_ANALYZE +# define ANALYZE 0 +#else +# define ANALYZE 0x00000004 +#endif +#ifdef SQLITE_OMIT_ATTACH +# define ATTACH 0 +#else +# define ATTACH 0x00000008 +#endif +#ifdef SQLITE_OMIT_AUTOINCREMENT +# define AUTOINCR 0 +#else +# define AUTOINCR 0x00000010 +#endif +#ifdef SQLITE_OMIT_CAST +# define CAST 0 +#else +# define CAST 0x00000020 +#endif +#ifdef SQLITE_OMIT_COMPOUND_SELECT +# define COMPOUND 0 +#else +# define COMPOUND 0x00000040 +#endif +#ifdef SQLITE_OMIT_CONFLICT_CLAUSE +# define CONFLICT 0 +#else +# define CONFLICT 0x00000080 +#endif +#ifdef SQLITE_OMIT_EXPLAIN +# define EXPLAIN 0 +#else +# define EXPLAIN 0x00000100 +#endif +#ifdef SQLITE_OMIT_FOREIGN_KEY +# define FKEY 0 +#else +# define FKEY 0x00000200 +#endif +#ifdef SQLITE_OMIT_PRAGMA +# define PRAGMA 0 +#else +# define PRAGMA 0x00000400 +#endif +#ifdef SQLITE_OMIT_REINDEX +# define REINDEX 0 +#else +# define REINDEX 0x00000800 +#endif +#ifdef SQLITE_OMIT_SUBQUERY +# define SUBQUERY 0 +#else +# define SUBQUERY 0x00001000 +#endif +#ifdef SQLITE_OMIT_TRIGGER +# define TRIGGER 0 +#else +# define TRIGGER 0x00002000 +#endif +#if defined(SQLITE_OMIT_AUTOVACUUM) && \ + (defined(SQLITE_OMIT_VACUUM) || defined(SQLITE_OMIT_ATTACH)) +# define VACUUM 0 +#else +# define VACUUM 0x00004000 +#endif +#ifdef SQLITE_OMIT_VIEW +# define VIEW 0 +#else +# define VIEW 0x00008000 +#endif +#ifdef SQLITE_OMIT_VIRTUALTABLE +# define VTAB 0 +#else +# define VTAB 0x00010000 +#endif +#ifdef SQLITE_OMIT_AUTOVACUUM +# define AUTOVACUUM 0 +#else +# define AUTOVACUUM 0x00020000 +#endif +#ifdef SQLITE_OMIT_CTE +# define CTE 0 +#else +# define CTE 0x00040000 +#endif +#ifdef SQLITE_OMIT_UPSERT +# define UPSERT 0 +#else +# define UPSERT 0x00080000 +#endif +#ifdef SQLITE_OMIT_WINDOWFUNC +# define WINDOWFUNC 0 +#else +# define WINDOWFUNC 0x00100000 +#endif +#ifdef SQLITE_OMIT_GENERATED_COLUMNS +# define GENCOL 0 +#else +# define GENCOL 0x00200000 +#endif +#ifdef SQLITE_OMIT_RETURNING +# define RETURNING 0 +#else +# define RETURNING 0x00400000 +#endif + + +/* +** These are the keywords +*/ +static Keyword aKeywordTable[] = { + { "ABORT", "TK_ABORT", CONFLICT|TRIGGER, 0 }, + { "ACTION", "TK_ACTION", FKEY, 0 }, + { "ADD", "TK_ADD", ALTER, 1 }, + { "AFTER", "TK_AFTER", TRIGGER, 0 }, + { "ALL", "TK_ALL", ALWAYS, 0 }, + { "ALTER", "TK_ALTER", ALTER, 0 }, + { "ALWAYS", "TK_ALWAYS", GENCOL, 0 }, + { "ANALYZE", "TK_ANALYZE", ANALYZE, 0 }, + { "AND", "TK_AND", ALWAYS, 10 }, + { "AS", "TK_AS", ALWAYS, 10 }, + { "ASC", "TK_ASC", ALWAYS, 0 }, + { "ATTACH", "TK_ATTACH", ATTACH, 1 }, + { "AUTOINCREMENT", "TK_AUTOINCR", AUTOINCR, 0 }, + { "BEFORE", "TK_BEFORE", TRIGGER, 0 }, + { "BEGIN", "TK_BEGIN", ALWAYS, 1 }, + { "BETWEEN", "TK_BETWEEN", ALWAYS, 5 }, + { "BY", "TK_BY", ALWAYS, 10 }, + { "CASCADE", "TK_CASCADE", FKEY, 1 }, + { "CASE", "TK_CASE", ALWAYS, 5 }, + { "CAST", "TK_CAST", CAST, 5 }, + { "CHECK", "TK_CHECK", ALWAYS, 1 }, + { "COLLATE", "TK_COLLATE", ALWAYS, 1 }, + { "COLUMN", "TK_COLUMNKW", ALTER, 1 }, + { "COMMIT", "TK_COMMIT", ALWAYS, 1 }, + { "CONFLICT", "TK_CONFLICT", CONFLICT, 0 }, + { "CONSTRAINT", "TK_CONSTRAINT", ALWAYS, 1 }, + { "CREATE", "TK_CREATE", ALWAYS, 2 }, + { "CROSS", "TK_JOIN_KW", ALWAYS, 3 }, + { "CURRENT", "TK_CURRENT", WINDOWFUNC, 1 }, + { "CURRENT_DATE", "TK_CTIME_KW", ALWAYS, 1 }, + { "CURRENT_TIME", "TK_CTIME_KW", ALWAYS, 1 }, + { "CURRENT_TIMESTAMP","TK_CTIME_KW", ALWAYS, 1 }, + { "DATABASE", "TK_DATABASE", ATTACH, 0 }, + { "DEFAULT", "TK_DEFAULT", ALWAYS, 1 }, + { "DEFERRED", "TK_DEFERRED", ALWAYS, 1 }, + { "DEFERRABLE", "TK_DEFERRABLE", FKEY, 1 }, + { "DELETE", "TK_DELETE", ALWAYS, 10 }, + { "DESC", "TK_DESC", ALWAYS, 3 }, + { "DETACH", "TK_DETACH", ATTACH, 0 }, + { "DISTINCT", "TK_DISTINCT", ALWAYS, 5 }, + { "DO", "TK_DO", UPSERT, 2 }, + { "DROP", "TK_DROP", ALWAYS, 1 }, + { "END", "TK_END", ALWAYS, 1 }, + { "EACH", "TK_EACH", TRIGGER, 1 }, + { "ELSE", "TK_ELSE", ALWAYS, 2 }, + { "ESCAPE", "TK_ESCAPE", ALWAYS, 4 }, + { "EXCEPT", "TK_EXCEPT", COMPOUND, 4 }, + { "EXCLUSIVE", "TK_EXCLUSIVE", ALWAYS, 1 }, + { "EXCLUDE", "TK_EXCLUDE", WINDOWFUNC, 1 }, + { "EXISTS", "TK_EXISTS", ALWAYS, 4 }, + { "EXPLAIN", "TK_EXPLAIN", EXPLAIN, 1 }, + { "FAIL", "TK_FAIL", CONFLICT|TRIGGER, 1 }, + { "FILTER", "TK_FILTER", WINDOWFUNC, 4 }, + { "FIRST", "TK_FIRST", ALWAYS, 4 }, + { "FOLLOWING", "TK_FOLLOWING", WINDOWFUNC, 4 }, + { "FOR", "TK_FOR", TRIGGER, 2 }, + { "FOREIGN", "TK_FOREIGN", FKEY, 1 }, + { "FROM", "TK_FROM", ALWAYS, 10 }, + { "FULL", "TK_JOIN_KW", ALWAYS, 3 }, + { "GENERATED", "TK_GENERATED", ALWAYS, 1 }, + { "GLOB", "TK_LIKE_KW", ALWAYS, 3 }, + { "GROUP", "TK_GROUP", ALWAYS, 5 }, + { "GROUPS", "TK_GROUPS", WINDOWFUNC, 2 }, + { "HAVING", "TK_HAVING", ALWAYS, 5 }, + { "IF", "TK_IF", ALWAYS, 2 }, + { "IGNORE", "TK_IGNORE", CONFLICT|TRIGGER, 1 }, + { "IMMEDIATE", "TK_IMMEDIATE", ALWAYS, 1 }, + { "IN", "TK_IN", ALWAYS, 10 }, + { "INDEX", "TK_INDEX", ALWAYS, 1 }, + { "INDEXED", "TK_INDEXED", ALWAYS, 0 }, + { "INITIALLY", "TK_INITIALLY", FKEY, 1 }, + { "INNER", "TK_JOIN_KW", ALWAYS, 1 }, + { "INSERT", "TK_INSERT", ALWAYS, 10 }, + { "INSTEAD", "TK_INSTEAD", TRIGGER, 1 }, + { "INTERSECT", "TK_INTERSECT", COMPOUND, 5 }, + { "INTO", "TK_INTO", ALWAYS, 10 }, + { "IS", "TK_IS", ALWAYS, 5 }, + { "ISNULL", "TK_ISNULL", ALWAYS, 5 }, + { "JOIN", "TK_JOIN", ALWAYS, 5 }, + { "KEY", "TK_KEY", ALWAYS, 1 }, + { "LAST", "TK_LAST", ALWAYS, 4 }, + { "LEFT", "TK_JOIN_KW", ALWAYS, 5 }, + { "LIKE", "TK_LIKE_KW", ALWAYS, 5 }, + { "LIMIT", "TK_LIMIT", ALWAYS, 3 }, + { "MATCH", "TK_MATCH", ALWAYS, 2 }, + { "MATERIALIZED", "TK_MATERIALIZED", CTE, 12 }, + { "NATURAL", "TK_JOIN_KW", ALWAYS, 3 }, + { "NO", "TK_NO", FKEY|WINDOWFUNC, 2 }, + { "NOT", "TK_NOT", ALWAYS, 10 }, + { "NOTHING", "TK_NOTHING", UPSERT, 1 }, + { "NOTNULL", "TK_NOTNULL", ALWAYS, 3 }, + { "NULL", "TK_NULL", ALWAYS, 10 }, + { "NULLS", "TK_NULLS", ALWAYS, 3 }, + { "OF", "TK_OF", ALWAYS, 3 }, + { "OFFSET", "TK_OFFSET", ALWAYS, 1 }, + { "ON", "TK_ON", ALWAYS, 1 }, + { "OR", "TK_OR", ALWAYS, 9 }, + { "ORDER", "TK_ORDER", ALWAYS, 10 }, + { "OTHERS", "TK_OTHERS", WINDOWFUNC, 3 }, + { "OUTER", "TK_JOIN_KW", ALWAYS, 5 }, + { "OVER", "TK_OVER", WINDOWFUNC, 3 }, + { "PARTITION", "TK_PARTITION", WINDOWFUNC, 3 }, + { "PLAN", "TK_PLAN", EXPLAIN, 0 }, + { "PRAGMA", "TK_PRAGMA", PRAGMA, 0 }, + { "PRECEDING", "TK_PRECEDING", WINDOWFUNC, 3 }, + { "PRIMARY", "TK_PRIMARY", ALWAYS, 1 }, + { "QUERY", "TK_QUERY", EXPLAIN, 0 }, + { "RAISE", "TK_RAISE", TRIGGER, 1 }, + { "RANGE", "TK_RANGE", WINDOWFUNC, 3 }, + { "RECURSIVE", "TK_RECURSIVE", CTE, 3 }, + { "REFERENCES", "TK_REFERENCES", FKEY, 1 }, + { "REGEXP", "TK_LIKE_KW", ALWAYS, 3 }, + { "REINDEX", "TK_REINDEX", REINDEX, 1 }, + { "RELEASE", "TK_RELEASE", ALWAYS, 1 }, + { "RENAME", "TK_RENAME", ALTER, 1 }, + { "REPLACE", "TK_REPLACE", CONFLICT, 10 }, + { "RESTRICT", "TK_RESTRICT", FKEY, 1 }, + { "RETURNING", "TK_RETURNING", RETURNING, 10 }, + { "RIGHT", "TK_JOIN_KW", ALWAYS, 0 }, + { "ROLLBACK", "TK_ROLLBACK", ALWAYS, 1 }, + { "ROW", "TK_ROW", TRIGGER, 1 }, + { "ROWS", "TK_ROWS", ALWAYS, 1 }, + { "SAVEPOINT", "TK_SAVEPOINT", ALWAYS, 1 }, + { "SELECT", "TK_SELECT", ALWAYS, 10 }, + { "SET", "TK_SET", ALWAYS, 10 }, + { "TABLE", "TK_TABLE", ALWAYS, 1 }, + { "TEMP", "TK_TEMP", ALWAYS, 1 }, + { "TEMPORARY", "TK_TEMP", ALWAYS, 1 }, + { "THEN", "TK_THEN", ALWAYS, 3 }, + { "TIES", "TK_TIES", WINDOWFUNC, 3 }, + { "TO", "TK_TO", ALWAYS, 3 }, + { "TRANSACTION", "TK_TRANSACTION", ALWAYS, 1 }, + { "TRIGGER", "TK_TRIGGER", TRIGGER, 1 }, + { "UNBOUNDED", "TK_UNBOUNDED", WINDOWFUNC, 3 }, + { "UNION", "TK_UNION", COMPOUND, 3 }, + { "UNIQUE", "TK_UNIQUE", ALWAYS, 1 }, + { "UPDATE", "TK_UPDATE", ALWAYS, 10 }, + { "USING", "TK_USING", ALWAYS, 8 }, + { "VACUUM", "TK_VACUUM", VACUUM, 1 }, + { "VALUES", "TK_VALUES", ALWAYS, 10 }, + { "VIEW", "TK_VIEW", VIEW, 1 }, + { "VIRTUAL", "TK_VIRTUAL", VTAB, 1 }, + { "WHEN", "TK_WHEN", ALWAYS, 1 }, + { "WHERE", "TK_WHERE", ALWAYS, 10 }, + { "WINDOW", "TK_WINDOW", WINDOWFUNC, 3 }, + { "WITH", "TK_WITH", CTE, 4 }, + { "WITHOUT", "TK_WITHOUT", ALWAYS, 1 }, +}; + +/* Number of keywords */ +static int nKeyword = (sizeof(aKeywordTable)/sizeof(aKeywordTable[0])); + +/* Map all alphabetic characters into lower-case for hashing. This is +** only valid for alphabetics. In particular it does not work for '_' +** and so the hash cannot be on a keyword position that might be an '_'. +*/ +#define charMap(X) (0x20|(X)) + +/* +** Comparision function for two Keyword records +*/ +static int keywordCompare1(const void *a, const void *b){ + const Keyword *pA = (Keyword*)a; + const Keyword *pB = (Keyword*)b; + int n = pA->len - pB->len; + if( n==0 ){ + n = strcmp(pA->zName, pB->zName); + } + assert( n!=0 ); + return n; +} +static int keywordCompare2(const void *a, const void *b){ + const Keyword *pA = (Keyword*)a; + const Keyword *pB = (Keyword*)b; + int n = pB->longestSuffix - pA->longestSuffix; + if( n==0 ){ + n = strcmp(pA->zName, pB->zName); + } + assert( n!=0 ); + return n; +} +static int keywordCompare3(const void *a, const void *b){ + const Keyword *pA = (Keyword*)a; + const Keyword *pB = (Keyword*)b; + int n = pA->offset - pB->offset; + if( n==0 ) n = pB->id - pA->id; + assert( n!=0 ); + return n; +} + +/* +** Return a KeywordTable entry with the given id +*/ +static Keyword *findById(int id){ + int i; + for(i=0; i<nKeyword; i++){ + if( aKeywordTable[i].id==id ) break; + } + return &aKeywordTable[i]; +} + +/* +** If aKeyword[*pFrom-1].iNext has a higher priority that aKeyword[*pFrom-1] +** itself, then swap them. +*/ +static void reorder(int *pFrom){ + int i = *pFrom - 1; + int j; + if( i<0 ) return; + j = aKeywordTable[i].iNext; + if( j==0 ) return; + j--; + if( aKeywordTable[i].priority >= aKeywordTable[j].priority ) return; + aKeywordTable[i].iNext = aKeywordTable[j].iNext; + aKeywordTable[j].iNext = i+1; + *pFrom = j+1; + reorder(&aKeywordTable[i].iNext); +} + +/* Parameter to the hash function +*/ +#define HASH_OP ^ +#define HASH_CC '^' +#define HASH_C0 4 +#define HASH_C1 3 +#define HASH_C2 1 + +/* +** This routine does the work. The generated code is printed on standard +** output. +*/ +int main(int argc, char **argv){ + int i, j, k, h; + int bestSize, bestCount; + int count; + int nChar; + int totalLen = 0; + int aKWHash[1000]; /* 1000 is much bigger than nKeyword */ + char zKWText[2000]; + + /* Remove entries from the list of keywords that have mask==0 */ + for(i=j=0; i<nKeyword; i++){ + if( aKeywordTable[i].mask==0 ) continue; + if( j<i ){ + aKeywordTable[j] = aKeywordTable[i]; + } + j++; + } + nKeyword = j; + + /* Fill in the lengths of strings and hashes for all entries. */ + for(i=0; i<nKeyword; i++){ + Keyword *p = &aKeywordTable[i]; + p->len = (int)strlen(p->zName); + assert( p->len<sizeof(p->zOrigName) ); + memcpy(p->zOrigName, p->zName, p->len+1); + totalLen += p->len; + p->hash = (charMap(p->zName[0])*HASH_C0) HASH_OP + (charMap(p->zName[p->len-1])*HASH_C1) HASH_OP + (p->len*HASH_C2); + p->id = i+1; + } + + /* Sort the table from shortest to longest keyword */ + qsort(aKeywordTable, nKeyword, sizeof(aKeywordTable[0]), keywordCompare1); + + /* Look for short keywords embedded in longer keywords */ + for(i=nKeyword-2; i>=0; i--){ + Keyword *p = &aKeywordTable[i]; + for(j=nKeyword-1; j>i && p->substrId==0; j--){ + Keyword *pOther = &aKeywordTable[j]; + if( pOther->substrId ) continue; + if( pOther->len<=p->len ) continue; + for(k=0; k<=pOther->len-p->len; k++){ + if( memcmp(p->zName, &pOther->zName[k], p->len)==0 ){ + p->substrId = pOther->id; + p->substrOffset = k; + break; + } + } + } + } + + /* Compute the longestSuffix value for every word */ + for(i=0; i<nKeyword; i++){ + Keyword *p = &aKeywordTable[i]; + if( p->substrId ) continue; + for(j=0; j<nKeyword; j++){ + Keyword *pOther; + if( j==i ) continue; + pOther = &aKeywordTable[j]; + if( pOther->substrId ) continue; + for(k=p->longestSuffix+1; k<p->len && k<pOther->len; k++){ + if( memcmp(&p->zName[p->len-k], pOther->zName, k)==0 ){ + p->longestSuffix = k; + } + } + } + } + + /* Sort the table into reverse order by length */ + qsort(aKeywordTable, nKeyword, sizeof(aKeywordTable[0]), keywordCompare2); + + /* Fill in the offset for all entries */ + nChar = 0; + for(i=0; i<nKeyword; i++){ + Keyword *p = &aKeywordTable[i]; + if( p->offset>0 || p->substrId ) continue; + p->offset = nChar; + nChar += p->len; + for(k=p->len-1; k>=1; k--){ + for(j=i+1; j<nKeyword; j++){ + Keyword *pOther = &aKeywordTable[j]; + if( pOther->offset>0 || pOther->substrId ) continue; + if( pOther->len<=k ) continue; + if( memcmp(&p->zName[p->len-k], pOther->zName, k)==0 ){ + p = pOther; + p->offset = nChar - k; + nChar = p->offset + p->len; + p->zName += k; + p->len -= k; + p->prefix = k; + j = i; + k = p->len; + } + } + } + } + for(i=0; i<nKeyword; i++){ + Keyword *p = &aKeywordTable[i]; + if( p->substrId ){ + p->offset = findById(p->substrId)->offset + p->substrOffset; + } + } + + /* Sort the table by offset */ + qsort(aKeywordTable, nKeyword, sizeof(aKeywordTable[0]), keywordCompare3); + + /* Figure out how big to make the hash table in order to minimize the + ** number of collisions */ + bestSize = nKeyword; + bestCount = nKeyword*nKeyword; + for(i=nKeyword/2; i<=2*nKeyword; i++){ + if( i<=0 ) continue; + for(j=0; j<i; j++) aKWHash[j] = 0; + for(j=0; j<nKeyword; j++){ + h = aKeywordTable[j].hash % i; + aKWHash[h] *= 2; + aKWHash[h]++; + } + for(j=count=0; j<i; j++) count += aKWHash[j]; + if( count<bestCount ){ + bestCount = count; + bestSize = i; + } + } + + /* Compute the hash */ + for(i=0; i<bestSize; i++) aKWHash[i] = 0; + for(i=0; i<nKeyword; i++){ + h = aKeywordTable[i].hash % bestSize; + aKeywordTable[i].iNext = aKWHash[h]; + aKWHash[h] = i+1; + reorder(&aKWHash[h]); + } + + /* Begin generating code */ + printf("%s", zHdr); + printf("/* Hash score: %d */\n", bestCount); + printf("/* zKWText[] encodes %d bytes of keyword text in %d bytes */\n", + totalLen + nKeyword, nChar+1 ); + for(i=j=k=0; i<nKeyword; i++){ + Keyword *p = &aKeywordTable[i]; + if( p->substrId ) continue; + memcpy(&zKWText[k], p->zName, p->len); + k += p->len; + if( j+p->len>70 ){ + printf("%*s */\n", 74-j, ""); + j = 0; + } + if( j==0 ){ + printf("/* "); + j = 8; + } + printf("%s", p->zName); + j += p->len; + } + if( j>0 ){ + printf("%*s */\n", 74-j, ""); + } + printf("static const char zKWText[%d] = {\n", nChar); + zKWText[nChar] = 0; + for(i=j=0; i<k; i++){ + if( j==0 ){ + printf(" "); + } + if( zKWText[i]==0 ){ + printf("0"); + }else{ + printf("'%c',", zKWText[i]); + } + j += 4; + if( j>68 ){ + printf("\n"); + j = 0; + } + } + if( j>0 ) printf("\n"); + printf("};\n"); + + printf("/* aKWHash[i] is the hash value for the i-th keyword */\n"); + printf("static const unsigned char aKWHash[%d] = {\n", bestSize); + for(i=j=0; i<bestSize; i++){ + if( j==0 ) printf(" "); + printf(" %3d,", aKWHash[i]); + j++; + if( j>12 ){ + printf("\n"); + j = 0; + } + } + printf("%s};\n", j==0 ? "" : "\n"); + + printf("/* aKWNext[] forms the hash collision chain. If aKWHash[i]==0\n"); + printf("** then the i-th keyword has no more hash collisions. Otherwise,\n"); + printf("** the next keyword with the same hash is aKWHash[i]-1. */\n"); + printf("static const unsigned char aKWNext[%d] = {0,\n", nKeyword+1); + for(i=j=0; i<nKeyword; i++){ + if( j==0 ) printf(" "); + printf(" %3d,", aKeywordTable[i].iNext); + j++; + if( j>12 ){ + printf("\n"); + j = 0; + } + } + printf("%s};\n", j==0 ? "" : "\n"); + + printf("/* aKWLen[i] is the length (in bytes) of the i-th keyword */\n"); + printf("static const unsigned char aKWLen[%d] = {0,\n", nKeyword+1); + for(i=j=0; i<nKeyword; i++){ + if( j==0 ) printf(" "); + printf(" %3d,", aKeywordTable[i].len+aKeywordTable[i].prefix); + j++; + if( j>12 ){ + printf("\n"); + j = 0; + } + } + printf("%s};\n", j==0 ? "" : "\n"); + + printf("/* aKWOffset[i] is the index into zKWText[] of the start of\n"); + printf("** the text for the i-th keyword. */\n"); + printf("static const unsigned short int aKWOffset[%d] = {0,\n", nKeyword+1); + for(i=j=0; i<nKeyword; i++){ + if( j==0 ) printf(" "); + printf(" %3d,", aKeywordTable[i].offset); + j++; + if( j>12 ){ + printf("\n"); + j = 0; + } + } + printf("%s};\n", j==0 ? "" : "\n"); + + printf("/* aKWCode[i] is the parser symbol code for the i-th keyword */\n"); + printf("static const unsigned char aKWCode[%d] = {0,\n", nKeyword+1); + for(i=j=0; i<nKeyword; i++){ + char *zToken = aKeywordTable[i].zTokenType; + if( j==0 ) printf(" "); + printf("%s,%*s", zToken, (int)(14-strlen(zToken)), ""); + j++; + if( j>=5 ){ + printf("\n"); + j = 0; + } + } + printf("%s};\n", j==0 ? "" : "\n"); + printf("/* Hash table decoded:\n"); + for(i=0; i<bestSize; i++){ + j = aKWHash[i]; + printf("** %3d:", i); + while( j ){ + printf(" %s", aKeywordTable[j-1].zOrigName); + j = aKeywordTable[j-1].iNext; + } + printf("\n"); + } + printf("*/\n"); + printf("/* Check to see if z[0..n-1] is a keyword. If it is, write the\n"); + printf("** parser symbol code for that keyword into *pType. Always\n"); + printf("** return the integer n (the length of the token). */\n"); + printf("static int keywordCode(const char *z, int n, int *pType){\n"); + printf(" int i, j;\n"); + printf(" const char *zKW;\n"); + printf(" assert( n>=2 );\n"); + printf(" i = ((charMap(z[0])*%d) %c", HASH_C0, HASH_CC); + printf(" (charMap(z[n-1])*%d) %c", HASH_C1, HASH_CC); + printf(" n*%d) %% %d;\n", HASH_C2, bestSize); + printf(" for(i=(int)aKWHash[i]; i>0; i=aKWNext[i]){\n"); + printf(" if( aKWLen[i]!=n ) continue;\n"); + printf(" zKW = &zKWText[aKWOffset[i]];\n"); + printf("#ifdef SQLITE_ASCII\n"); + printf(" if( (z[0]&~0x20)!=zKW[0] ) continue;\n"); + printf(" if( (z[1]&~0x20)!=zKW[1] ) continue;\n"); + printf(" j = 2;\n"); + printf(" while( j<n && (z[j]&~0x20)==zKW[j] ){ j++; }\n"); + printf("#endif\n"); + printf("#ifdef SQLITE_EBCDIC\n"); + printf(" if( toupper(z[0])!=zKW[0] ) continue;\n"); + printf(" if( toupper(z[1])!=zKW[1] ) continue;\n"); + printf(" j = 2;\n"); + printf(" while( j<n && toupper(z[j])==zKW[j] ){ j++; }\n"); + printf("#endif\n"); + printf(" if( j<n ) continue;\n"); + for(i=0; i<nKeyword; i++){ + printf(" testcase( i==%d ); /* %s */\n", + i+1, aKeywordTable[i].zOrigName); + } + printf(" *pType = aKWCode[i];\n"); + printf(" break;\n"); + printf(" }\n"); + printf(" return n;\n"); + printf("}\n"); + printf("int sqlite3KeywordCode(const unsigned char *z, int n){\n"); + printf(" int id = TK_ID;\n"); + printf(" if( n>=2 ) keywordCode((char*)z, n, &id);\n"); + printf(" return id;\n"); + printf("}\n"); + printf("#define SQLITE_N_KEYWORD %d\n", nKeyword); + printf("int sqlite3_keyword_name(int i,const char **pzName,int *pnName){\n"); + printf(" if( i<0 || i>=SQLITE_N_KEYWORD ) return SQLITE_ERROR;\n"); + printf(" i++;\n"); + printf(" *pzName = zKWText + aKWOffset[i];\n"); + printf(" *pnName = aKWLen[i];\n"); + printf(" return SQLITE_OK;\n"); + printf("}\n"); + printf("int sqlite3_keyword_count(void){ return SQLITE_N_KEYWORD; }\n"); + printf("int sqlite3_keyword_check(const char *zName, int nName){\n"); + printf(" return TK_ID!=sqlite3KeywordCode((const u8*)zName, nName);\n"); + printf("}\n"); + + return 0; +} diff --git a/tool/mkmsvcmin.tcl b/tool/mkmsvcmin.tcl new file mode 100644 index 0000000..6cb145d --- /dev/null +++ b/tool/mkmsvcmin.tcl @@ -0,0 +1,113 @@ +#!/usr/bin/tcl +# +# This script reads the regular MSVC makefile (../Makefile.msc) and outputs +# a revised version of that Makefile that is "minimal" in the sense that +# it uses the sqlite3.c amalgamation as input and does not require tclsh. +# The resulting "../Makefile.min.msc" is suitable for use in the amalgamation +# tarballs. +# +if {$argc==0} { + set basedir [file dir [file dir [file normalize $argv0]]] + set fromFileName [file join $basedir Makefile.msc] + set toFileName [file join $basedir autoconf Makefile.msc] +} else { + set fromFileName [lindex $argv 0] + if {![file exists $fromFileName]} { + error "input file \"$fromFileName\" does not exist" + } + set toFileName [lindex $argv 1] + if {[file exists $toFileName]} { + error "output file \"$toFileName\" already exists" + } +} + +proc readFile { fileName } { + set file_id [open $fileName RDONLY] + fconfigure $file_id -encoding binary -translation binary + set result [read $file_id] + close $file_id + return $result +} + +proc writeFile { fileName data } { + set file_id [open $fileName {WRONLY CREAT TRUNC}] + fconfigure $file_id -encoding binary -translation binary + puts -nonewline $file_id $data + close $file_id + return "" +} + +proc escapeSubSpec { data } { + regsub -all -- {&} $data {\\\&} data + regsub -all -- {\\(\d+)} $data {\\\\\1} data + return $data +} + +proc substVars { data } { + return [uplevel 1 [list subst -nocommands -nobackslashes $data]] +} + +# +# NOTE: This block is used to replace the section marked <<block1>> in +# the Makefile, if it exists. +# +set blocks(1) [string trimleft [string map [list \\\\ \\] { +_HASHCHAR=^# +!IF ![echo !IFNDEF VERSION > rcver.vc] && \\ + ![for /F "delims=" %V in ('type "$(SQLITE3H)" ^| "%SystemRoot%\System32\find.exe" "$(_HASHCHAR)define SQLITE_VERSION "') do (echo VERSION = ^^%V >> rcver.vc)] && \\ + ![echo !ENDIF >> rcver.vc] +!INCLUDE rcver.vc +!ENDIF + +RESOURCE_VERSION = $(VERSION:^#=) +RESOURCE_VERSION = $(RESOURCE_VERSION:define=) +RESOURCE_VERSION = $(RESOURCE_VERSION:SQLITE_VERSION=) +RESOURCE_VERSION = $(RESOURCE_VERSION:"=) +RESOURCE_VERSION = $(RESOURCE_VERSION:.=,) + +$(LIBRESOBJS): $(TOP)\sqlite3.rc rcver.vc $(SQLITE3H) + echo #ifndef SQLITE_RESOURCE_VERSION > sqlite3rc.h + echo #define SQLITE_RESOURCE_VERSION $(RESOURCE_VERSION) >> sqlite3rc.h + echo #endif >> sqlite3rc.h + $(LTRCOMPILE) -fo $(LIBRESOBJS) -DRC_VERONLY $(TOP)\sqlite3.rc +}]] + +# +# NOTE: This block is used to replace the section marked <<block2>> in +# the Makefile, if it exists. +# +set blocks(2) [string trimleft [string map [list \\\\ \\] { +Replace.exe: + $(CSC) /target:exe $(TOP)\Replace.cs + +sqlite3.def: Replace.exe $(LIBOBJ) + echo EXPORTS > sqlite3.def + dumpbin /all $(LIBOBJ) \\ + | .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset|changegroup|rebaser|rbu)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \\ + | sort >> sqlite3.def +}]] + +set data "#### DO NOT EDIT ####\n" +append data "# This makefile is automatically " +append data "generated from the [file tail $fromFileName] at\n" +append data "# the root of the canonical SQLite source tree (not the\n" +append data "# amalgamation tarball) using the tool/[file tail $argv0]\n" +append data "# script.\n#\n\n" +append data [readFile $fromFileName] + +regsub -all -- {# <<mark>>\n.*?# <</mark>>\n} \ + $data "" data + +foreach i [lsort -integer [array names blocks]] { + regsub -all -- [substVars \ + {# <<block${i}>>\n.*?# <</block${i}>>\n}] \ + $data [escapeSubSpec $blocks($i)] data +} + +set data [string map [list " -I\$(TOP)\\src" ""] $data] +set data [string map [list " libsqlite3.lib" ""] $data] +set data [string map [list " \$(ALL_TCL_TARGETS)" ""] $data] +set data [string map [list "\$(TOP)\\src\\" "\$(TOP)\\"] $data] + +writeFile $toFileName $data +puts "generated $toFileName from $fromFileName" diff --git a/tool/mkopcodec.tcl b/tool/mkopcodec.tcl new file mode 100644 index 0000000..5eac05f --- /dev/null +++ b/tool/mkopcodec.tcl @@ -0,0 +1,51 @@ +#!/usr/bin/tclsh +# +# This TCL script scans the opcodes.h file (which is itself generated by +# another TCL script) and uses the information gleaned to create the +# opcodes.c source file. +# +# Opcodes.c contains strings which are the symbolic names for the various +# opcodes used by the VDBE. These strings are used when disassembling a +# VDBE program during tracing or as a result of the EXPLAIN keyword. +# +puts "/* Automatically generated. Do not edit */" +puts "/* See the tool/mkopcodec.tcl script for details. */" +puts "#if !defined(SQLITE_OMIT_EXPLAIN) \\" +puts " || defined(VDBE_PROFILE) \\" +puts " || defined(SQLITE_DEBUG)" +puts "#if defined(SQLITE_ENABLE_EXPLAIN_COMMENTS) || defined(SQLITE_DEBUG)" +puts "# define OpHelp(X) \"\\0\" X" +puts "#else" +puts "# define OpHelp(X)" +puts "#endif" +puts "const char *sqlite3OpcodeName(int i)\173" +puts " static const char *const azName\[\] = \173" +set mx 0 + +set in [open [lindex $argv 0]] +fconfigure $in -translation binary +while {![eof $in]} { + set line [gets $in] + if {[regexp {^#define OP_} $line]} { + set name [lindex $line 1] + regsub {^OP_} $name {} name + set i [lindex $line 2] + set label($i) $name + if {$mx<$i} {set mx $i} + if {[regexp {synopsis: (.*) \*/} $line all x]} { + set synopsis($i) [string trim $x] + } else { + set synopsis($i) {} + } + } +} +close $in + +for {set i 0} {$i<=$mx} {incr i} { + puts [format " /* %3d */ %-18s OpHelp(\"%s\")," \ + $i \"$label($i)\" $synopsis($i)] +} +puts " \175;" +puts " return azName\[i\];" +puts "\175" +puts "#endif" diff --git a/tool/mkopcodeh.tcl b/tool/mkopcodeh.tcl new file mode 100644 index 0000000..6fb3b75 --- /dev/null +++ b/tool/mkopcodeh.tcl @@ -0,0 +1,325 @@ +#!/usr/bin/tclsh +# +# Generate the file opcodes.h. +# +# This TCL script scans a concatenation of the parse.h output file from the +# parser and the vdbe.c source file in order to generate the opcodes numbers +# for all opcodes. +# +# The lines of the vdbe.c that we are interested in are of the form: +# +# case OP_aaaa: /* same as TK_bbbbb */ +# +# The TK_ comment is optional. If it is present, then the value assigned to +# the OP_ is the same as the TK_ value. If missing, the OP_ value is assigned +# a small integer that is different from every other OP_ value. +# +# We go to the trouble of making some OP_ values the same as TK_ values +# as an optimization. During parsing, things like expression operators +# are coded with TK_ values such as TK_ADD, TK_DIVIDE, and so forth. Later +# during code generation, we need to generate corresponding opcodes like +# OP_Add and OP_Divide. By making TK_ADD==OP_Add and TK_DIVIDE==OP_Divide, +# code to translate from one to the other is avoided. This makes the +# code generator smaller and faster. +# +# This script also scans for lines of the form: +# +# case OP_aaaa: /* jump, in1, in2, in3, out2, out3 */ +# +# When such comments are found on an opcode, it means that certain +# properties apply to that opcode. Set corresponding flags using the +# OPFLG_INITIALIZER macro. +# + +set in stdin +set currentOp {} +set prevName {} +set nOp 0 +set nGroup 0 +while {![eof $in]} { + set line [gets $in] + + # Remember the TK_ values from the parse.h file. + # NB: The "TK_" prefix stands for "ToKen", not the graphical Tk toolkit + # commonly associated with TCL. + # + if {[regexp {^#define TK_} $line]} { + set tk([lindex $line 1]) [lindex $line 2] + continue + } + + # Find "/* Opcode: " lines in the vdbe.c file. Each one introduces + # a new opcode. Remember which parameters are used. + # + if {[regexp {^.. Opcode: } $line]} { + set currentOp OP_[lindex $line 2] + set m 0 + foreach term $line { + switch $term { + P1 {incr m 1} + P2 {incr m 2} + P3 {incr m 4} + P4 {incr m 8} + P5 {incr m 16} + } + } + set paramused($currentOp) $m + } + + # Find "** Synopsis: " lines that follow Opcode: + # + if {[regexp {^.. Synopsis: (.*)} $line all x] && $currentOp!=""} { + set synopsis($currentOp) [string trim $x] + } + + # Scan for "case OP_aaaa:" lines in the vdbe.c file + # + if {[regexp {^case OP_} $line]} { + set line [split $line] + set name [string trim [lindex $line 1] :] + if {$name=="OP_Abortable"} continue; # put OP_Abortable last + set op($name) -1 + set group($name) 0 + set jump($name) 0 + set in1($name) 0 + set in2($name) 0 + set in3($name) 0 + set out2($name) 0 + set out3($name) 0 + set ncycle($name) 0 + for {set i 3} {$i<[llength $line]-1} {incr i} { + switch [string trim [lindex $line $i] ,] { + same { + incr i + if {[lindex $line $i]=="as"} { + incr i + set sym [string trim [lindex $line $i] ,] + set val $tk($sym) + set op($name) $val + set used($val) 1 + set sameas($val) $sym + set def($val) $name + } + } + group {set group($name) 1} + jump {set jump($name) 1} + in1 {set in1($name) 1} + in2 {set in2($name) 1} + in3 {set in3($name) 1} + out2 {set out2($name) 1} + out3 {set out3($name) 1} + ncycle {set ncycle($name) 1} + } + } + if {$group($name)} { + set newGroup 0 + if {[info exists groups($nGroup)]} { + if {$prevName=="" || !$group($prevName)} { + set newGroup 1 + } + } + lappend groups($nGroup) $name + if {$newGroup} {incr nGroup} + } else { + if {$prevName!="" && $group($prevName)} { + incr nGroup + } + } + set order($nOp) $name + set prevName $name + incr nOp + } +} + +# Assign numbers to all opcodes and output the result. +# +puts "/* Automatically generated. Do not edit */" +puts "/* See the tool/mkopcodeh.tcl script for details */" +foreach name {OP_Noop OP_Explain OP_Abortable} { + set jump($name) 0 + set in1($name) 0 + set in2($name) 0 + set in3($name) 0 + set out2($name) 0 + set out3($name) 0 + set ncycle($name) 0 + set op($name) -1 + set order($nOp) $name + incr nOp +} + +# The following are the opcodes that receive special processing in the +# resolveP2Values() routine. Update this list whenever new cases are +# added to the pOp->opcode switch within resolveP2Values(). +# +set rp2v_ops { + OP_Transaction + OP_AutoCommit + OP_Savepoint + OP_Checkpoint + OP_Vacuum + OP_JournalMode + OP_VUpdate + OP_VFilter + OP_Init +} + +# Assign the smallest values to opcodes that are processed by resolveP2Values() +# to make code generation for the switch() statement smaller and faster. +# +set cnt -1 +for {set i 0} {$i<$nOp} {incr i} { + set name $order($i) + if {[lsearch $rp2v_ops $name]>=0} { + incr cnt + while {[info exists used($cnt)]} {incr cnt} + set op($name) $cnt + set used($cnt) 1 + set def($cnt) $name + } +} +set mxCase1 $cnt + +# Assign the next group of values to JUMP opcodes +# +for {set i 0} {$i<$nOp} {incr i} { + set name $order($i) + if {$op($name)>=0} continue + if {!$jump($name)} continue + incr cnt + while {[info exists used($cnt)]} {incr cnt} + set op($name) $cnt + set used($cnt) 1 + set def($cnt) $name +} + +# Find the numeric value for the largest JUMP opcode +# +set mxJump -1 +for {set i 0} {$i<$nOp} {incr i} { + set name $order($i) + if {$jump($name) && $op($name)>$mxJump} {set mxJump $op($name)} +} + + +# Generate the numeric values for all remaining opcodes, while +# preserving any groupings of opcodes (i.e. those that must be +# together). +# +for {set g 0} {$g<$nGroup} {incr g} { + set gLen [llength $groups($g)] + set ok 0; set start -1 + set seek $cnt + while {!$ok} { + incr seek + while {[info exists used($seek)]} {incr seek} + set ok 1; set start $seek + for {set j 0} {$j<$gLen} {incr j} { + incr seek + if {[info exists used($seek)]} { + set ok 0; break + } + } + } + if {$ok} { + set next $start + for {set j 0} {$j<$gLen} {incr j} { + set name [lindex $groups($g) $j] + if {$op($name)>=0} continue + set op($name) $next + set used($next) 1 + set def($next) $name + incr next + } + } else { + error "cannot find opcodes for group: $groups($g)" + } +} + +for {set i 0} {$i<$nOp} {incr i} { + set name $order($i) + if {$op($name)<0} { + incr cnt + while {[info exists used($cnt)]} {incr cnt} + set op($name) $cnt + set used($cnt) 1 + set def($cnt) $name + } +} + +set max [lindex [lsort -decr -integer [array names used]] 0] +for {set i 0} {$i<=$max} {incr i} { + if {![info exists used($i)]} { + set def($i) "OP_NotUsed_$i" + } + if {$i>$max} {set max $i} + set name $def($i) + puts -nonewline [format {#define %-16s %3d} $name $i] + set com {} + if {[info exists jump($name)] && $jump($name)} { + lappend com "jump" + } + if {[info exists sameas($i)]} { + lappend com "same as $sameas($i)" + } + if {[info exists synopsis($name)]} { + lappend com "synopsis: $synopsis($name)" + } + if {[llength $com]} { + puts -nonewline [format " /* %-42s */" [join $com {, }]] + } + puts "" +} + +if {$max>255} { + error "More than 255 opcodes - VdbeOp.opcode is of type u8!" +} + +# Generate the bitvectors: +# +set bv(0) 0 +for {set i 0} {$i<=$max} {incr i} { + set x 0 + set name $def($i) + if {[string match OP_NotUsed* $name]==0} { + if {$jump($name)} {incr x 1} + if {$in1($name)} {incr x 2} + if {$in2($name)} {incr x 4} + if {$in3($name)} {incr x 8} + if {$out2($name)} {incr x 16} + if {$out3($name)} {incr x 32} + if {$ncycle($name)} {incr x 64} + } + set bv($i) $x +} +puts "" +puts "/* Properties such as \"out2\" or \"jump\" that are specified in" +puts "** comments following the \"case\" for each opcode in the vdbe.c" +puts "** are encoded into bitvectors as follows:" +puts "*/" +puts "#define OPFLG_JUMP 0x01 /* jump: P2 holds jmp target */" +puts "#define OPFLG_IN1 0x02 /* in1: P1 is an input */" +puts "#define OPFLG_IN2 0x04 /* in2: P2 is an input */" +puts "#define OPFLG_IN3 0x08 /* in3: P3 is an input */" +puts "#define OPFLG_OUT2 0x10 /* out2: P2 is an output */" +puts "#define OPFLG_OUT3 0x20 /* out3: P3 is an output */" +puts "#define OPFLG_NCYCLE 0x40 /* ncycle:Cycles count against P1 */" +puts "#define OPFLG_INITIALIZER \173\\" +for {set i 0} {$i<=$max} {incr i} { + if {$i%8==0} { + puts -nonewline [format "/* %3d */" $i] + } + puts -nonewline [format " 0x%02x," $bv($i)] + if {$i%8==7} { + puts "\\" + } +} +puts "\175" +puts "" +puts "/* The resolve3P2Values() routine is able to run faster if it knows" +puts "** the value of the largest JUMP opcode. The smaller the maximum" +puts "** JUMP opcode the better, so the mkopcodeh.tcl script that" +puts "** generated this include file strives to group all JUMP opcodes" +puts "** together near the beginning of the list." +puts "*/" +puts "#define SQLITE_MX_JUMP_OPCODE $mxJump /* Maximum JUMP opcode */" diff --git a/tool/mkopts.tcl b/tool/mkopts.tcl new file mode 100644 index 0000000..88f645b --- /dev/null +++ b/tool/mkopts.tcl @@ -0,0 +1,51 @@ +#!/usr/bin/tclsh +# +# This script is used to generate the array of strings and the enum +# that appear at the beginning of the C code implementation of a +# a TCL command and that define the available subcommands for that +# TCL command. + +set prefix {} +while {![eof stdin]} { + set line [gets stdin] + if {$line==""} continue + regsub -all "\[ \t\n,\]+" [string trim $line] { } line + foreach token [split $line { }] { + if {![regexp {(([a-zA-Z]+)_)?([_a-zA-Z0-9]+)} $token all px p2 name]} continue + lappend namelist [string tolower $name] + if {$px!=""} {set prefix $p2} + } +} + +puts " static const char *${prefix}_strs\[\] = \173" +set col 0 +proc put_item x { + global col + if {$col==0} {puts -nonewline " "} + if {$col<2} { + puts -nonewline [format " %-25s" $x] + incr col + } else { + puts $x + set col 0 + } +} +proc finalize {} { + global col + if {$col>0} {puts {}} + set col 0 +} + +foreach name [lsort $namelist] { + put_item \"$name\", +} +put_item 0 +finalize +puts " \175;" +puts " enum ${prefix}_enum \173" +foreach name [lsort $namelist] { + regsub -all {@} $name {} name + put_item ${prefix}_[string toupper $name], +} +finalize +puts " \175;" diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl new file mode 100644 index 0000000..2b5b6fd --- /dev/null +++ b/tool/mkpragmatab.tcl @@ -0,0 +1,637 @@ +#!/usr/bin/tclsh +# +# Run this script to generate the pragma name lookup table C code. +# +# To add new pragmas, first add the name and other relevant attributes +# of the pragma to the "pragma_def" object below. Then run this script +# to generate the ../src/pragma.h header file that contains macros and +# the lookup table needed for pragma name lookup in the pragma.c module. +# Then add the extra "case PragTyp_XXXXX:" and subsequent code for the +# new pragma in ../src/pragma.c. +# +# The results are normally written into the ../src/pragma.h file. However, +# if an alternative output file name is provided as an argument, then +# results are written into the alternative. For example: +# +# tclsh tool/mkpragmatab.tcl ;# <--- Results to src/pragma.h +# +# tclsh tool/mkpragmatab.tcl /dev/tty ;# <-- results to terminal +# + +# Flag meanings: +set flagMeaning(NeedSchema) {Force schema load before running} +set flagMeaning(ReadOnly) {Read-only HEADER_VALUE} +set flagMeaning(Result0) {Acts as query when no argument} +set flagMeaning(Result1) {Acts as query when has one argument} +set flagMeaning(SchemaReq) {Schema required - "main" is default} +set flagMeaning(SchemaOpt) {Schema restricts name search if present} +set flagMeaning(NoColumns) {OP_ResultRow called with zero columns} +set flagMeaning(NoColumns1) {zero columns if RHS argument is present} + +set pragma_def { + NAME: full_column_names + TYPE: FLAG + ARG: SQLITE_FullColNames + IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) + + NAME: short_column_names + TYPE: FLAG + ARG: SQLITE_ShortColNames + IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) + + NAME: count_changes + TYPE: FLAG + ARG: SQLITE_CountRows + IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) + + NAME: empty_result_callbacks + TYPE: FLAG + ARG: SQLITE_NullCallback + IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) + + NAME: fullfsync + TYPE: FLAG + ARG: SQLITE_FullFSync + IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) + + NAME: checkpoint_fullfsync + TYPE: FLAG + ARG: SQLITE_CkptFullFSync + IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) + + NAME: cache_spill + FLAG: Result0 SchemaReq NoColumns1 + IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) + + NAME: reverse_unordered_selects + TYPE: FLAG + ARG: SQLITE_ReverseOrder + IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) + + NAME: query_only + TYPE: FLAG + ARG: SQLITE_QueryOnly + IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) + + NAME: automatic_index + TYPE: FLAG + ARG: SQLITE_AutoIndex + IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) + IF: !defined(SQLITE_OMIT_AUTOMATIC_INDEX) + + NAME: sql_trace + TYPE: FLAG + ARG: SQLITE_SqlTrace + IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) + IF: defined(SQLITE_DEBUG) + + NAME: vdbe_listing + TYPE: FLAG + ARG: SQLITE_VdbeListing + IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) + IF: defined(SQLITE_DEBUG) + + NAME: vdbe_trace + TYPE: FLAG + ARG: SQLITE_VdbeTrace + IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) + IF: defined(SQLITE_DEBUG) + + NAME: vdbe_addoptrace + TYPE: FLAG + ARG: SQLITE_VdbeAddopTrace + IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) + IF: defined(SQLITE_DEBUG) + + NAME: vdbe_debug + TYPE: FLAG + ARG: SQLITE_SqlTrace|SQLITE_VdbeListing|SQLITE_VdbeTrace + IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) + IF: defined(SQLITE_DEBUG) + + NAME: vdbe_eqp + TYPE: FLAG + ARG: SQLITE_VdbeEQP + IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) + IF: defined(SQLITE_DEBUG) + + NAME: ignore_check_constraints + TYPE: FLAG + ARG: SQLITE_IgnoreChecks + IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) + IF: !defined(SQLITE_OMIT_CHECK) + + NAME: writable_schema + TYPE: FLAG + ARG: SQLITE_WriteSchema|SQLITE_NoSchemaError + IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) + + NAME: read_uncommitted + TYPE: FLAG + ARG: SQLITE_ReadUncommit + IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) + + NAME: recursive_triggers + TYPE: FLAG + ARG: SQLITE_RecTriggers + IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) + + NAME: trusted_schema + TYPE: FLAG + ARG: SQLITE_TrustedSchema + IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) + + NAME: foreign_keys + TYPE: FLAG + ARG: SQLITE_ForeignKeys + IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) + IF: !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) + + NAME: defer_foreign_keys + TYPE: FLAG + ARG: SQLITE_DeferFKs + IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) + IF: !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) + + NAME: cell_size_check + TYPE: FLAG + ARG: SQLITE_CellSizeCk + + NAME: default_cache_size + FLAG: NeedSchema Result0 SchemaReq NoColumns1 + COLS: cache_size + IF: !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) + + NAME: page_size + FLAG: Result0 SchemaReq NoColumns1 + IF: !defined(SQLITE_OMIT_PAGER_PRAGMAS) + + NAME: secure_delete + FLAG: Result0 + IF: !defined(SQLITE_OMIT_PAGER_PRAGMAS) + + NAME: page_count + FLAG: NeedSchema Result0 SchemaReq + IF: !defined(SQLITE_OMIT_PAGER_PRAGMAS) + + NAME: max_page_count + TYPE: PAGE_COUNT + FLAG: NeedSchema Result0 SchemaReq + IF: !defined(SQLITE_OMIT_PAGER_PRAGMAS) + + NAME: locking_mode + FLAG: Result0 SchemaReq + IF: !defined(SQLITE_OMIT_PAGER_PRAGMAS) + + NAME: journal_mode + FLAG: NeedSchema Result0 SchemaReq + IF: !defined(SQLITE_OMIT_PAGER_PRAGMAS) + + NAME: journal_size_limit + FLAG: Result0 SchemaReq + IF: !defined(SQLITE_OMIT_PAGER_PRAGMAS) + + NAME: cache_size + FLAG: NeedSchema Result0 SchemaReq NoColumns1 + IF: !defined(SQLITE_OMIT_PAGER_PRAGMAS) + + NAME: mmap_size + IF: !defined(SQLITE_OMIT_PAGER_PRAGMAS) + + NAME: auto_vacuum + FLAG: NeedSchema Result0 SchemaReq NoColumns1 + IF: !defined(SQLITE_OMIT_AUTOVACUUM) + + NAME: incremental_vacuum + FLAG: NeedSchema NoColumns + IF: !defined(SQLITE_OMIT_AUTOVACUUM) + + NAME: temp_store + FLAG: Result0 NoColumns1 + IF: !defined(SQLITE_OMIT_PAGER_PRAGMAS) + + NAME: temp_store_directory + FLAG: NoColumns1 + IF: !defined(SQLITE_OMIT_PAGER_PRAGMAS) + + NAME: data_store_directory + FLAG: NoColumns1 + IF: !defined(SQLITE_OMIT_PAGER_PRAGMAS) && SQLITE_OS_WIN + + NAME: lock_proxy_file + FLAG: NoColumns1 + IF: !defined(SQLITE_OMIT_PAGER_PRAGMAS) && SQLITE_ENABLE_LOCKING_STYLE + + NAME: synchronous + FLAG: NeedSchema Result0 SchemaReq NoColumns1 + IF: !defined(SQLITE_OMIT_PAGER_PRAGMAS) + + NAME: table_info + FLAG: NeedSchema Result1 SchemaOpt + ARG: 0 + COLS: cid name type notnull dflt_value pk + IF: !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + + NAME: table_xinfo + TYPE: TABLE_INFO + FLAG: NeedSchema Result1 SchemaOpt + ARG: 1 + COLS: cid name type notnull dflt_value pk hidden + IF: !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + + NAME: table_list + TYPE: TABLE_LIST + FLAG: NeedSchema Result1 + COLS: schema name type ncol wr strict + IF: !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + + NAME: stats + FLAG: NeedSchema Result0 SchemaReq + COLS: tbl idx wdth hght flgs + IF: !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) && defined(SQLITE_DEBUG) + + NAME: index_info + TYPE: INDEX_INFO + ARG: 0 + FLAG: NeedSchema Result1 SchemaOpt + COLS: seqno cid name + IF: !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + + NAME: index_xinfo + TYPE: INDEX_INFO + ARG: 1 + FLAG: NeedSchema Result1 SchemaOpt + COLS: seqno cid name desc coll key + IF: !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + + NAME: index_list + FLAG: NeedSchema Result1 SchemaOpt + COLS: seq name unique origin partial + IF: !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + + NAME: database_list + FLAG: Result0 + COLS: seq name file + IF: !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + + NAME: function_list + FLAG: Result0 + COLS: name builtin type enc narg flags + IF: !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + IF: !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) + + NAME: module_list + FLAG: Result0 + COLS: name + IF: !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + IF: !defined(SQLITE_OMIT_VIRTUALTABLE) + IF: !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) + + NAME: pragma_list + FLAG: Result0 + COLS: name + IF: !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) + + NAME: collation_list + FLAG: Result0 + COLS: seq name + IF: !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + + NAME: foreign_key_list + FLAG: NeedSchema Result1 SchemaOpt + COLS: id seq table from to on_update on_delete match + IF: !defined(SQLITE_OMIT_FOREIGN_KEY) + + NAME: foreign_key_check + FLAG: NeedSchema Result0 Result1 SchemaOpt + COLS: table rowid parent fkid + IF: !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) + + NAME: parser_trace + TYPE: FLAG + ARG: SQLITE_ParserTrace + IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) + IF: defined(SQLITE_DEBUG) + + NAME: case_sensitive_like + FLAG: NoColumns + IF: !defined(SQLITE_OMIT_CASE_SENSITIVE_LIKE_PRAGMA) + + NAME: integrity_check + FLAG: NeedSchema Result0 Result1 SchemaOpt + IF: !defined(SQLITE_OMIT_INTEGRITY_CHECK) + + NAME: quick_check + TYPE: INTEGRITY_CHECK + FLAG: NeedSchema Result0 Result1 SchemaOpt + IF: !defined(SQLITE_OMIT_INTEGRITY_CHECK) + + NAME: encoding + FLAG: Result0 NoColumns1 + IF: !defined(SQLITE_OMIT_UTF16) + + NAME: schema_version + TYPE: HEADER_VALUE + ARG: BTREE_SCHEMA_VERSION + FLAG: NoColumns1 Result0 + IF: !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + + NAME: user_version + TYPE: HEADER_VALUE + ARG: BTREE_USER_VERSION + FLAG: NoColumns1 Result0 + IF: !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + + NAME: data_version + TYPE: HEADER_VALUE + ARG: BTREE_DATA_VERSION + FLAG: ReadOnly Result0 + IF: !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + + NAME: freelist_count + TYPE: HEADER_VALUE + ARG: BTREE_FREE_PAGE_COUNT + FLAG: ReadOnly Result0 + IF: !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + + NAME: application_id + TYPE: HEADER_VALUE + ARG: BTREE_APPLICATION_ID + FLAG: NoColumns1 Result0 + IF: !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + + NAME: compile_options + FLAG: Result0 + IF: !defined(SQLITE_OMIT_COMPILEOPTION_DIAGS) + + NAME: wal_checkpoint + FLAG: NeedSchema + COLS: busy log checkpointed + IF: !defined(SQLITE_OMIT_WAL) + + NAME: wal_autocheckpoint + IF: !defined(SQLITE_OMIT_WAL) + + NAME: shrink_memory + FLAG: NoColumns + + NAME: busy_timeout + FLAG: Result0 + COLS: timeout + + NAME: lock_status + FLAG: Result0 + COLS: database status + IF: defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + + NAME: activate_extensions + IF: defined(SQLITE_ENABLE_CEROD) + + NAME: soft_heap_limit + FLAG: Result0 + + NAME: hard_heap_limit + FLAG: Result0 + + NAME: threads + FLAG: Result0 + + NAME: analysis_limit + FLAG: Result0 + + NAME: optimize + FLAG: Result1 NeedSchema + + NAME: legacy_alter_table + TYPE: FLAG + ARG: SQLITE_LegacyAlter + IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) +} + +# Open the output file +# +if {$argc>0} { + set destfile [lindex $argv 0] +} else { + set destfile "[file dir [file dir [file normal $argv0]]]/src/pragma.h" + puts "Overwriting $destfile with new pragma table..." +} +set fd [open $destfile wb] +puts $fd {/* DO NOT EDIT! +** This file is automatically generated by the script at +** ../tool/mkpragmatab.tcl. To update the set of pragmas, edit +** that script and rerun it. +*/} + +# Parse the PRAGMA table above. +# +set name {} +set type {} +set if {} +set flags {} +set cols {} +set cols_list {} +set arg 0 +proc record_one {} { + global name type if arg allbyname typebyif flags cols all_cols + global cols_list colUsedBy + if {$name==""} return + if {$cols!=""} { + if {![info exists all_cols($cols)]} { + set all_cols($cols) 1 + lappend cols_list $cols + } + set cx $cols + lappend colUsedBy($cols) $name + } else { + set cx 0 + } + set allbyname($name) [list $type $arg $if $flags $cols] + set name {} + set type {} + set if {} + set flags {} + set cols {} + set arg 0 +} +foreach line [split $pragma_def \n] { + set line [string trim $line] + if {$line==""} continue + foreach {id val} [split $line :] break + set val [string trim $val] + if {$id=="NAME"} { + record_one + set name $val + set type [string toupper $val] + } elseif {$id=="TYPE"} { + set type $val + if {$type=="FLAG"} { + lappend flags Result0 NoColumns1 + } + } elseif {$id=="ARG"} { + set arg $val + } elseif {$id=="COLS"} { + set cols $val + } elseif {$id=="IF"} { + lappend if $val + } elseif {$id=="FLAG"} { + foreach term [split $val] { + lappend flags $term + set allflags($term) 1 + } + } else { + error "bad pragma_def line: $line" + } +} +record_one +set allnames [lsort [array names allbyname]] + +# Generate #defines for all pragma type names. Group the pragmas that are +# omit in default builds (ex: defined(SQLITE_DEBUG)) +# at the end. +# +puts $fd "\n/* The various pragma types */" +set pnum 0 +foreach name $allnames { + set type [lindex $allbyname($name) 0] + if {[info exists seentype($type)]} continue + set if [lindex $allbyname($name) 2] + if {[regexp SQLITE_DEBUG $if] || [regexp SQLITE_HAS_CODEC $if]} continue + set seentype($type) 1 + puts $fd [format {#define %-35s %4d} PragTyp_$type $pnum] + incr pnum +} +foreach name $allnames { + set type [lindex $allbyname($name) 0] + if {[info exists seentype($type)]} continue + set if [lindex $allbyname($name) 2] + if {[regexp SQLITE_DEBUG $if]} continue + set seentype($type) 1 + puts $fd [format {#define %-35s %4d} PragTyp_$type $pnum] + incr pnum +} +foreach name $allnames { + set type [lindex $allbyname($name) 0] + if {[info exists seentype($type)]} continue + set seentype($type) 1 + puts $fd [format {#define %-35s %4d} PragTyp_$type $pnum] + incr pnum +} + +# Generate #defines for flags +# +puts $fd "\n/* Property flags associated with various pragma. */" +set fv 1 +foreach f [lsort [array names allflags]] { + puts $fd [format {#define PragFlg_%-10s 0x%02x /* %s */} \ + $f $fv $flagMeaning($f)] + set fv [expr {$fv*2}] +} + +# Sort the column lists so that longer column lists occur first +# +proc colscmp {a b} { + return [expr {[llength $b] - [llength $a]}] +} +set cols_list [lsort -command colscmp $cols_list] + +# Generate the array of column names used by pragmas that act like +# queries. +# +puts $fd "\n/* Names of columns for pragmas that return multi-column result" +puts $fd "** or that return single-column results where the name of the" +puts $fd "** result column is different from the name of the pragma\n*/" +puts $fd "static const char *const pragCName\[\] = {" +set offset 0 +set allcollist {} +foreach cols $cols_list { + set n [llength $cols] + set limit [expr {[llength $allcollist] - $n}] + for {set i 0} {$i<$limit} {incr i} { + set sublist [lrange $allcollist $i [expr {$i+$n-1}]] + if {$sublist==$cols} { + puts $fd [format "%27s/* $colUsedBy($cols) reuses $i */" ""] + set cols_offset($cols) $i + break + } + } + if {$i<$limit} continue + set cols_offset($cols) $offset + set ub " /* Used by: $colUsedBy($cols) */" + foreach c $cols { + lappend allcollist $c + puts $fd [format " /* %3d */ %-14s%s" $offset \"$c\", $ub] + set ub "" + incr offset + } +} +puts $fd "\175;" + +# Generate the lookup table +# +puts $fd "\n/* Definitions of all built-in pragmas */" +puts $fd "typedef struct PragmaName \173" +puts $fd " const char *const zName; /* Name of pragma */" +puts $fd " u8 ePragTyp; /* PragTyp_XXX value */" +puts $fd " u8 mPragFlg; /* Zero or more PragFlg_XXX values */" +puts $fd { u8 iPragCName; /* Start of column names in pragCName[] */} +puts $fd " u8 nPragCName; \ +/* Num of col names. 0 means use pragma name */" +puts $fd " u64 iArg; /* Extra argument */" +puts $fd "\175 PragmaName;" +puts $fd "static const PragmaName aPragmaName\[\] = \173" + +set current_if {} +set spacer [format { %26s } {}] +foreach name $allnames { + foreach {type arg if flag cx} $allbyname($name) break + if {$cx==0 || $cx==""} { + set cy 0 + set nx 0 + } else { + set cy $cols_offset($cx) + set nx [llength $cx] + } + if {$if!=$current_if} { + if {$current_if!=""} { + foreach this_if $current_if { + puts $fd "#endif" + } + } + set current_if $if + if {$current_if!=""} { + foreach this_if $current_if { + puts $fd "#if $this_if" + } + } + } + set typex [format PragTyp_%-23s $type,] + if {$flag==""} { + set flagx "0" + } else { + set flagx PragFlg_[join $flag {|PragFlg_}] + } + puts $fd " \173/* zName: */ \"$name\"," + puts $fd " /* ePragTyp: */ PragTyp_$type," + puts $fd " /* ePragFlg: */ $flagx," + puts $fd " /* ColNames: */ $cy, $nx," + puts $fd " /* iArg: */ $arg \175," +} +if {$current_if!=""} { + foreach this_if $current_if { + puts $fd "#endif" + } +} +puts $fd "\175;" + +# count the number of pragmas, for information purposes +# +set allcnt 0 +set dfltcnt 0 +foreach name $allnames { + incr allcnt + set if [lindex $allbyname($name) 2] + if {[regexp {^defined} $if] || [regexp {[^!]defined} $if]} continue + incr dfltcnt +} +puts $fd "/* Number of pragmas: $dfltcnt on by default, $allcnt total. */" diff --git a/tool/mkshellc.tcl b/tool/mkshellc.tcl new file mode 100644 index 0000000..4f16a77 --- /dev/null +++ b/tool/mkshellc.tcl @@ -0,0 +1,75 @@ +#!/usr/bin/tclsh +# +# Run this script to generate the "shell.c" source file from +# constituent parts. +# +# No arguments are required. This script determines the location +# of its input files relative to the location of the script itself. +# This script should be tool/mkshellc.tcl. If the directory holding +# the script is $DIR, then the component parts are located in $DIR/../src +# and $DIR/../ext/misc. +# +set topdir [file dir [file dir [file normal $argv0]]] +set out stdout +fconfigure stdout -translation binary +puts $out {/* DO NOT EDIT! +** This file is automatically generated by the script in the canonical +** SQLite source tree at tool/mkshellc.tcl. That script combines source +** code from various constituent source files of SQLite into this single +** "shell.c" file used to implement the SQLite command-line shell. +** +** Most of the code found below comes from the "src/shell.c.in" file in +** the canonical SQLite source tree. That main file contains "INCLUDE" +** lines that specify other files in the canonical source tree that are +** inserted to getnerate this complete program source file. +** +** The code from multiple files is combined into this single "shell.c" +** source file to help make the command-line program easier to compile. +** +** To modify this program, get a copy of the canonical SQLite source tree, +** edit the src/shell.c.in" and/or some of the other files that are included +** by "src/shell.c.in", then rerun the tool/mkshellc.tcl script. +*/} +set in [open $topdir/src/shell.c.in] +fconfigure $in -translation binary +proc omit_redundant_typedefs {line} { + global typedef_seen + if {[regexp {^typedef .*\y([a-zA-Z0-9_]+);} $line all typename]} { + if {[info exists typedef_seen($typename)]} { + return "/* [string map {/* // */ //} $line] */" + } + set typedef_seen($typename) 1 + } + return $line +} +set iLine 0 +while {1} { + set lx [omit_redundant_typedefs [gets $in]] + if {[eof $in]} break; + incr iLine + if {[regexp {^INCLUDE } $lx]} { + set cfile [lindex $lx 1] + puts $out "/************************* Begin $cfile ******************/" +# puts $out "#line 1 \"$cfile\"" + set in2 [open $topdir/src/$cfile] + fconfigure $in2 -translation binary + while {![eof $in2]} { + set lx [omit_redundant_typedefs [gets $in2]] + if {[regexp {^# *include "sqlite} $lx]} { + set lx "/* $lx */" + } + if {[regexp {^# *include "test_windirent.h"} $lx]} { + set lx "/* $lx */" + } + set lx [string map [list __declspec(dllexport) {}] $lx] + puts $out $lx + } + close $in2 + puts $out "/************************* End $cfile ********************/" +# puts $out "#line [expr $iLine+1] \"shell.c.in\"" + continue + } + puts $out $lx +} +close $in +close $out diff --git a/tool/mksourceid.c b/tool/mksourceid.c new file mode 100644 index 0000000..dd15399 --- /dev/null +++ b/tool/mksourceid.c @@ -0,0 +1,835 @@ +/* +** Run this program with a single argument which is the name of the +** Fossil "manifest" file for a project, and this program will emit on +** standard output the "source id" for for the program. +** +** (1) The "source id" is the date of check-in together with the +** SHA3 hash of the manifest file. +** +** (2) All individual file hashes in the manifest are verified. If any +** source file has changed, the SHA3 hash ends with "modified". +** +*/ +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <sys/types.h> +#include <ctype.h> + +/* Portable 64-bit unsigned integers */ +#if defined(_MSC_VER) || defined(__BORLANDC__) + typedef unsigned __int64 u64; +#else + typedef unsigned long long int u64; +#endif + + +/* +** Macros to determine whether the machine is big or little endian, +** and whether or not that determination is run-time or compile-time. +** +** For best performance, an attempt is made to guess at the byte-order +** using C-preprocessor macros. If that is unsuccessful, or if +** -DBYTEORDER=0 is set, then byte-order is determined +** at run-time. +*/ +#ifndef BYTEORDER +# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__arm__) +# define BYTEORDER 1234 +# elif defined(sparc) || defined(__ppc__) +# define BYTEORDER 4321 +# else +# define BYTEORDER 0 +# endif +#endif + + + +/* +** State structure for a SHA3 hash in progress +*/ +typedef struct SHA3Context SHA3Context; +struct SHA3Context { + union { + u64 s[25]; /* Keccak state. 5x5 lines of 64 bits each */ + unsigned char x[1600]; /* ... or 1600 bytes */ + } u; + unsigned nRate; /* Bytes of input accepted per Keccak iteration */ + unsigned nLoaded; /* Input bytes loaded into u.x[] so far this cycle */ + unsigned ixMask; /* Insert next input into u.x[nLoaded^ixMask]. */ +}; + +/* +** A single step of the Keccak mixing function for a 1600-bit state +*/ +static void KeccakF1600Step(SHA3Context *p){ + int i; + u64 B0, B1, B2, B3, B4; + u64 C0, C1, C2, C3, C4; + u64 D0, D1, D2, D3, D4; + static const u64 RC[] = { + 0x0000000000000001ULL, 0x0000000000008082ULL, + 0x800000000000808aULL, 0x8000000080008000ULL, + 0x000000000000808bULL, 0x0000000080000001ULL, + 0x8000000080008081ULL, 0x8000000000008009ULL, + 0x000000000000008aULL, 0x0000000000000088ULL, + 0x0000000080008009ULL, 0x000000008000000aULL, + 0x000000008000808bULL, 0x800000000000008bULL, + 0x8000000000008089ULL, 0x8000000000008003ULL, + 0x8000000000008002ULL, 0x8000000000000080ULL, + 0x000000000000800aULL, 0x800000008000000aULL, + 0x8000000080008081ULL, 0x8000000000008080ULL, + 0x0000000080000001ULL, 0x8000000080008008ULL + }; +# define A00 (p->u.s[0]) +# define A01 (p->u.s[1]) +# define A02 (p->u.s[2]) +# define A03 (p->u.s[3]) +# define A04 (p->u.s[4]) +# define A10 (p->u.s[5]) +# define A11 (p->u.s[6]) +# define A12 (p->u.s[7]) +# define A13 (p->u.s[8]) +# define A14 (p->u.s[9]) +# define A20 (p->u.s[10]) +# define A21 (p->u.s[11]) +# define A22 (p->u.s[12]) +# define A23 (p->u.s[13]) +# define A24 (p->u.s[14]) +# define A30 (p->u.s[15]) +# define A31 (p->u.s[16]) +# define A32 (p->u.s[17]) +# define A33 (p->u.s[18]) +# define A34 (p->u.s[19]) +# define A40 (p->u.s[20]) +# define A41 (p->u.s[21]) +# define A42 (p->u.s[22]) +# define A43 (p->u.s[23]) +# define A44 (p->u.s[24]) +# define ROL64(a,x) ((a<<x)|(a>>(64-x))) + + for(i=0; i<24; i+=4){ + C0 = A00^A10^A20^A30^A40; + C1 = A01^A11^A21^A31^A41; + C2 = A02^A12^A22^A32^A42; + C3 = A03^A13^A23^A33^A43; + C4 = A04^A14^A24^A34^A44; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A11^D1), 44); + B2 = ROL64((A22^D2), 43); + B3 = ROL64((A33^D3), 21); + B4 = ROL64((A44^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i]; + A11 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A20^D0), 3); + B3 = ROL64((A31^D1), 45); + B4 = ROL64((A42^D2), 61); + B0 = ROL64((A03^D3), 28); + B1 = ROL64((A14^D4), 20); + A20 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A40^D0), 18); + B0 = ROL64((A01^D1), 1); + B1 = ROL64((A12^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A34^D4), 8); + A40 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A10^D0), 36); + B2 = ROL64((A21^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A43^D3), 56); + B0 = ROL64((A04^D4), 27); + A10 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A30^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A02^D2), 62); + B1 = ROL64((A13^D3), 55); + B2 = ROL64((A24^D4), 39); + A30 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + C0 = A00^A20^A40^A10^A30; + C1 = A11^A31^A01^A21^A41; + C2 = A22^A42^A12^A32^A02; + C3 = A33^A03^A23^A43^A13; + C4 = A44^A14^A34^A04^A24; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A31^D1), 44); + B2 = ROL64((A12^D2), 43); + B3 = ROL64((A43^D3), 21); + B4 = ROL64((A24^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+1]; + A31 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A40^D0), 3); + B3 = ROL64((A21^D1), 45); + B4 = ROL64((A02^D2), 61); + B0 = ROL64((A33^D3), 28); + B1 = ROL64((A14^D4), 20); + A40 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A30^D0), 18); + B0 = ROL64((A11^D1), 1); + B1 = ROL64((A42^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A04^D4), 8); + A30 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A20^D0), 36); + B2 = ROL64((A01^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A13^D3), 56); + B0 = ROL64((A44^D4), 27); + A20 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A10^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A22^D2), 62); + B1 = ROL64((A03^D3), 55); + B2 = ROL64((A34^D4), 39); + A10 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + C0 = A00^A40^A30^A20^A10; + C1 = A31^A21^A11^A01^A41; + C2 = A12^A02^A42^A32^A22; + C3 = A43^A33^A23^A13^A03; + C4 = A24^A14^A04^A44^A34; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A21^D1), 44); + B2 = ROL64((A42^D2), 43); + B3 = ROL64((A13^D3), 21); + B4 = ROL64((A34^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+2]; + A21 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A30^D0), 3); + B3 = ROL64((A01^D1), 45); + B4 = ROL64((A22^D2), 61); + B0 = ROL64((A43^D3), 28); + B1 = ROL64((A14^D4), 20); + A30 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A10^D0), 18); + B0 = ROL64((A31^D1), 1); + B1 = ROL64((A02^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A44^D4), 8); + A10 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A40^D0), 36); + B2 = ROL64((A11^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A03^D3), 56); + B0 = ROL64((A24^D4), 27); + A40 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A20^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A12^D2), 62); + B1 = ROL64((A33^D3), 55); + B2 = ROL64((A04^D4), 39); + A20 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + C0 = A00^A30^A10^A40^A20; + C1 = A21^A01^A31^A11^A41; + C2 = A42^A22^A02^A32^A12; + C3 = A13^A43^A23^A03^A33; + C4 = A34^A14^A44^A24^A04; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A01^D1), 44); + B2 = ROL64((A02^D2), 43); + B3 = ROL64((A03^D3), 21); + B4 = ROL64((A04^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+3]; + A01 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A10^D0), 3); + B3 = ROL64((A11^D1), 45); + B4 = ROL64((A12^D2), 61); + B0 = ROL64((A13^D3), 28); + B1 = ROL64((A14^D4), 20); + A10 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A20^D0), 18); + B0 = ROL64((A21^D1), 1); + B1 = ROL64((A22^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A24^D4), 8); + A20 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A30^D0), 36); + B2 = ROL64((A31^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A33^D3), 56); + B0 = ROL64((A34^D4), 27); + A30 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A40^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A42^D2), 62); + B1 = ROL64((A43^D3), 55); + B2 = ROL64((A44^D4), 39); + A40 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + } +} + +/* +** Initialize a new hash. iSize determines the size of the hash +** in bits and should be one of 224, 256, 384, or 512. Or iSize +** can be zero to use the default hash size of 256 bits. +*/ +static void SHA3Init(SHA3Context *p, int iSize){ + memset(p, 0, sizeof(*p)); + if( iSize>=128 && iSize<=512 ){ + p->nRate = (1600 - ((iSize + 31)&~31)*2)/8; + }else{ + p->nRate = (1600 - 2*256)/8; + } +#if BYTEORDER==1234 + /* Known to be little-endian at compile-time. No-op */ +#elif BYTEORDER==4321 + p->ixMask = 7; /* Big-endian */ +#else + { + static unsigned int one = 1; + if( 1==*(unsigned char*)&one ){ + /* Little endian. No byte swapping. */ + p->ixMask = 0; + }else{ + /* Big endian. Byte swap. */ + p->ixMask = 7; + } + } +#endif +} + +/* +** Make consecutive calls to the SHA3Update function to add new content +** to the hash +*/ +static void SHA3Update( + SHA3Context *p, + const unsigned char *aData, + unsigned int nData +){ + unsigned int i = 0; +#if BYTEORDER==1234 + if( (p->nLoaded % 8)==0 && ((aData - (const unsigned char*)0)&7)==0 ){ + for(; i+7<nData; i+=8){ + p->u.s[p->nLoaded/8] ^= *(u64*)&aData[i]; + p->nLoaded += 8; + if( p->nLoaded>=p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } + } +#endif + for(; i<nData; i++){ +#if BYTEORDER==1234 + p->u.x[p->nLoaded] ^= aData[i]; +#elif BYTEORDER==4321 + p->u.x[p->nLoaded^0x07] ^= aData[i]; +#else + p->u.x[p->nLoaded^p->ixMask] ^= aData[i]; +#endif + p->nLoaded++; + if( p->nLoaded==p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } +} + +/* +** After all content has been added, invoke SHA3Final() to compute +** the final hash. The function returns a pointer to the binary +** hash value. +*/ +static unsigned char *SHA3Final(SHA3Context *p){ + unsigned int i; + if( p->nLoaded==p->nRate-1 ){ + const unsigned char c1 = 0x86; + SHA3Update(p, &c1, 1); + }else{ + const unsigned char c2 = 0x06; + const unsigned char c3 = 0x80; + SHA3Update(p, &c2, 1); + p->nLoaded = p->nRate - 1; + SHA3Update(p, &c3, 1); + } + for(i=0; i<p->nRate; i++){ + p->u.x[i+p->nRate] = p->u.x[i^p->ixMask]; + } + return &p->u.x[p->nRate]; +} + +/* +** Convert a digest into base-16. digest should be declared as +** "unsigned char digest[20]" in the calling function. The SHA3 +** digest is stored in the first 20 bytes. zBuf should +** be "char zBuf[41]". +*/ +static void DigestToBase16(unsigned char *digest, char *zBuf, int nByte){ + static const char zEncode[] = "0123456789abcdef"; + int ix; + + for(ix=0; ix<nByte; ix++){ + *zBuf++ = zEncode[(*digest>>4)&0xf]; + *zBuf++ = zEncode[*digest++ & 0xf]; + } + *zBuf = '\0'; +} + + +/* +** Compute the SHA3 checksum of a file on disk. Store the resulting +** checksum in the blob pCksum. pCksum is assumed to be initialized. +** +** Return the number of errors. +*/ +static int sha3sum_file(const char *zFilename, int iSize, char *pCksum){ + FILE *in; + SHA3Context ctx; + char zBuf[10240]; + + in = fopen(zFilename,"rb"); + if( in==0 ){ + return 1; + } + SHA3Init(&ctx, iSize); + for(;;){ + int n = (int)fread(zBuf, 1, sizeof(zBuf), in); + if( n<=0 ) break; + SHA3Update(&ctx, (unsigned char*)zBuf, (unsigned)n); + } + fclose(in); + DigestToBase16(SHA3Final(&ctx), pCksum, iSize/8); + return 0; +} + +/* +** The SHA1 implementation below is adapted from: +** +** $NetBSD: sha1.c,v 1.6 2009/11/06 20:31:18 joerg Exp $ +** $OpenBSD: sha1.c,v 1.9 1997/07/23 21:12:32 kstailey Exp $ +** +** SHA-1 in C +** By Steve Reid <steve@edmweb.com> +** 100% Public Domain +*/ +typedef struct SHA1Context SHA1Context; +struct SHA1Context { + unsigned int state[5]; + unsigned int count[2]; + unsigned char buffer[64]; +}; + +/* + * blk0() and blk() perform the initial expand. + * I got the idea of expanding during the round function from SSLeay + * + * blk0le() for little-endian and blk0be() for big-endian. + */ +#define SHA_ROT(x,l,r) ((x) << (l) | (x) >> (r)) +#define rol(x,k) SHA_ROT(x,k,32-(k)) +#define ror(x,k) SHA_ROT(x,32-(k),k) + +#define blk0le(i) (block[i] = (ror(block[i],8)&0xFF00FF00) \ + |(rol(block[i],8)&0x00FF00FF)) +#define blk0be(i) block[i] +#define blk(i) (block[i&15] = rol(block[(i+13)&15]^block[(i+8)&15] \ + ^block[(i+2)&15]^block[i&15],1)) + +/* + * (R0+R1), R2, R3, R4 are the different operations (rounds) used in SHA1 + * + * Rl0() for little-endian and Rb0() for big-endian. Endianness is + * determined at run-time. + */ +#define Rl0(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk0le(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define Rb0(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk0be(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define R1(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define R2(v,w,x,y,z,i) \ + z+=(w^x^y)+blk(i)+0x6ED9EBA1+rol(v,5);w=ror(w,2); +#define R3(v,w,x,y,z,i) \ + z+=(((w|x)&y)|(w&x))+blk(i)+0x8F1BBCDC+rol(v,5);w=ror(w,2); +#define R4(v,w,x,y,z,i) \ + z+=(w^x^y)+blk(i)+0xCA62C1D6+rol(v,5);w=ror(w,2); + +/* + * Hash a single 512-bit block. This is the core of the algorithm. + */ +#define a qq[0] +#define b qq[1] +#define c qq[2] +#define d qq[3] +#define e qq[4] + +static void SHA1Transform( + unsigned int state[5], + const unsigned char buffer[64] +){ + unsigned int qq[5]; /* a, b, c, d, e; */ + static int one = 1; + unsigned int block[16]; + memcpy(block, buffer, 64); + memcpy(qq,state,5*sizeof(unsigned int)); + + /* Copy context->state[] to working vars */ + /* + a = state[0]; + b = state[1]; + c = state[2]; + d = state[3]; + e = state[4]; + */ + + /* 4 rounds of 20 operations each. Loop unrolled. */ + if( 1 == *(unsigned char*)&one ){ + Rl0(a,b,c,d,e, 0); Rl0(e,a,b,c,d, 1); Rl0(d,e,a,b,c, 2); Rl0(c,d,e,a,b, 3); + Rl0(b,c,d,e,a, 4); Rl0(a,b,c,d,e, 5); Rl0(e,a,b,c,d, 6); Rl0(d,e,a,b,c, 7); + Rl0(c,d,e,a,b, 8); Rl0(b,c,d,e,a, 9); Rl0(a,b,c,d,e,10); Rl0(e,a,b,c,d,11); + Rl0(d,e,a,b,c,12); Rl0(c,d,e,a,b,13); Rl0(b,c,d,e,a,14); Rl0(a,b,c,d,e,15); + }else{ + Rb0(a,b,c,d,e, 0); Rb0(e,a,b,c,d, 1); Rb0(d,e,a,b,c, 2); Rb0(c,d,e,a,b, 3); + Rb0(b,c,d,e,a, 4); Rb0(a,b,c,d,e, 5); Rb0(e,a,b,c,d, 6); Rb0(d,e,a,b,c, 7); + Rb0(c,d,e,a,b, 8); Rb0(b,c,d,e,a, 9); Rb0(a,b,c,d,e,10); Rb0(e,a,b,c,d,11); + Rb0(d,e,a,b,c,12); Rb0(c,d,e,a,b,13); Rb0(b,c,d,e,a,14); Rb0(a,b,c,d,e,15); + } + R1(e,a,b,c,d,16); R1(d,e,a,b,c,17); R1(c,d,e,a,b,18); R1(b,c,d,e,a,19); + R2(a,b,c,d,e,20); R2(e,a,b,c,d,21); R2(d,e,a,b,c,22); R2(c,d,e,a,b,23); + R2(b,c,d,e,a,24); R2(a,b,c,d,e,25); R2(e,a,b,c,d,26); R2(d,e,a,b,c,27); + R2(c,d,e,a,b,28); R2(b,c,d,e,a,29); R2(a,b,c,d,e,30); R2(e,a,b,c,d,31); + R2(d,e,a,b,c,32); R2(c,d,e,a,b,33); R2(b,c,d,e,a,34); R2(a,b,c,d,e,35); + R2(e,a,b,c,d,36); R2(d,e,a,b,c,37); R2(c,d,e,a,b,38); R2(b,c,d,e,a,39); + R3(a,b,c,d,e,40); R3(e,a,b,c,d,41); R3(d,e,a,b,c,42); R3(c,d,e,a,b,43); + R3(b,c,d,e,a,44); R3(a,b,c,d,e,45); R3(e,a,b,c,d,46); R3(d,e,a,b,c,47); + R3(c,d,e,a,b,48); R3(b,c,d,e,a,49); R3(a,b,c,d,e,50); R3(e,a,b,c,d,51); + R3(d,e,a,b,c,52); R3(c,d,e,a,b,53); R3(b,c,d,e,a,54); R3(a,b,c,d,e,55); + R3(e,a,b,c,d,56); R3(d,e,a,b,c,57); R3(c,d,e,a,b,58); R3(b,c,d,e,a,59); + R4(a,b,c,d,e,60); R4(e,a,b,c,d,61); R4(d,e,a,b,c,62); R4(c,d,e,a,b,63); + R4(b,c,d,e,a,64); R4(a,b,c,d,e,65); R4(e,a,b,c,d,66); R4(d,e,a,b,c,67); + R4(c,d,e,a,b,68); R4(b,c,d,e,a,69); R4(a,b,c,d,e,70); R4(e,a,b,c,d,71); + R4(d,e,a,b,c,72); R4(c,d,e,a,b,73); R4(b,c,d,e,a,74); R4(a,b,c,d,e,75); + R4(e,a,b,c,d,76); R4(d,e,a,b,c,77); R4(c,d,e,a,b,78); R4(b,c,d,e,a,79); + + /* Add the working vars back into context.state[] */ + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; + state[4] += e; +} + + +/* + * SHA1Init - Initialize new context + */ +static void SHA1Init(SHA1Context *context){ + /* SHA1 initialization constants */ + context->state[0] = 0x67452301; + context->state[1] = 0xEFCDAB89; + context->state[2] = 0x98BADCFE; + context->state[3] = 0x10325476; + context->state[4] = 0xC3D2E1F0; + context->count[0] = context->count[1] = 0; +} + + +/* + * Run your data through this. + */ +static void SHA1Update( + SHA1Context *context, + const unsigned char *data, + unsigned int len +){ + unsigned int i, j; + + j = context->count[0]; + if ((context->count[0] += len << 3) < j) + context->count[1] += (len>>29)+1; + j = (j >> 3) & 63; + if ((j + len) > 63) { + (void)memcpy(&context->buffer[j], data, (i = 64-j)); + SHA1Transform(context->state, context->buffer); + for ( ; i + 63 < len; i += 64) + SHA1Transform(context->state, &data[i]); + j = 0; + } else { + i = 0; + } + (void)memcpy(&context->buffer[j], &data[i], len - i); +} + + +/* + * Add padding and return the message digest. + */ +static void SHA1Final(unsigned char *digest, SHA1Context *context){ + unsigned int i; + unsigned char finalcount[8]; + + for (i = 0; i < 8; i++) { + finalcount[i] = (unsigned char)((context->count[(i >= 4 ? 0 : 1)] + >> ((3-(i & 3)) * 8) ) & 255); /* Endian independent */ + } + SHA1Update(context, (const unsigned char *)"\200", 1); + while ((context->count[0] & 504) != 448) + SHA1Update(context, (const unsigned char *)"\0", 1); + SHA1Update(context, finalcount, 8); /* Should cause a SHA1Transform() */ + + if (digest) { + for (i = 0; i < 20; i++) + digest[i] = (unsigned char) + ((context->state[i>>2] >> ((3-(i & 3)) * 8) ) & 255); + } +} + + +/* +** Compute the SHA1 checksum of a file on disk. Store the resulting +** checksum in the blob pCksum. pCksum is assumed to be initialized. +** +** Return the number of errors. +*/ +static int sha1sum_file(const char *zFilename, char *pCksum){ + FILE *in; + SHA1Context ctx; + unsigned char zResult[20]; + char zBuf[10240]; + + in = fopen(zFilename,"rb"); + if( in==0 ){ + return 1; + } + SHA1Init(&ctx); + for(;;){ + int n = (int)fread(zBuf, 1, sizeof(zBuf), in); + if( n<=0 ) break; + SHA1Update(&ctx, (unsigned char*)zBuf, (unsigned)n); + } + fclose(in); + SHA1Final(zResult, &ctx); + DigestToBase16(zResult, pCksum, 20); + return 0; +} + +/* +** Print a usage comment and quit. +*/ +static void usage(const char *argv0){ + fprintf(stderr, + "Usage: %s manifest\n" + "Options:\n" + " -v Diagnostic output\n" + , argv0); + exit(1); +} + +/* +** Find the first whitespace character in a string. Set that whitespace +** to a \000 terminator and return a pointer to the next character. +*/ +static char *nextToken(char *z){ + while( *z && !isspace(*z) ) z++; + if( *z==0 ) return z; + *z = 0; + return &z[1]; +} + + +int main(int argc, char **argv){ + const char *zManifest = 0; + int i; + int bVerbose = 0; + FILE *in; + int allValid = 1; + int rc; + SHA3Context ctx; + char zDate[50]; + char zHash[100]; + char zLine[20000]; + + for(i=1; i<argc; i++){ + const char *z = argv[i]; + if( z[0]=='-' ){ + if( z[1]=='-' ) z++; + if( strcmp(z, "-v")==0 ){ + bVerbose = 1; + }else + { + fprintf(stderr, "unknown option \"%s\"", argv[i]); + exit(1); + } + }else if( zManifest!=0 ){ + usage(argv[0]); + }else{ + zManifest = z; + } + } + if( zManifest==0 ) usage(argv[0]); + zDate[0] = 0; + in = fopen(zManifest, "rb"); + if( in==0 ){ + fprintf(stderr, "cannot open \"%s\" for reading\n", zManifest); + exit(1); + } + SHA3Init(&ctx, 256); + while( fgets(zLine, sizeof(zLine), in) ){ + if( strncmp(zLine,"# Remove this line", 18)!=0 ){ + SHA3Update(&ctx, (unsigned char*)zLine, (unsigned)strlen(zLine)); + } + if( strncmp(zLine, "D 20", 4)==0 ){ + memcpy(zDate, &zLine[2], 10); + zDate[10] = ' '; + memcpy(&zDate[11], &zLine[13], 8); + zDate[19] = 0; + continue; + } + if( strncmp(zLine, "F ", 2)==0 ){ + char *zFilename = &zLine[2]; + char *zMHash = nextToken(zFilename); + nextToken(zMHash); + if( strlen(zMHash)==40 ){ + rc = sha1sum_file(zFilename, zHash); + }else{ + rc = sha3sum_file(zFilename, 256, zHash); + } + if( rc ){ + allValid = 0; + if( bVerbose ){ + printf("hash failed: %s\n", zFilename); + } + }else if( strcmp(zHash, zMHash)!=0 ){ + allValid = 0; + if( bVerbose ){ + printf("wrong hash: %s\n", zFilename); + printf("... expected: %s\n", zMHash); + printf("... got: %s\n", zHash); + } + } + } + } + fclose(in); + DigestToBase16(SHA3Final(&ctx), zHash, 256/8); + if( !allValid ){ + printf("%s %.60salt1\n", zDate, zHash); + }else{ + printf("%s %s\n", zDate, zHash); + } + return 0; +} diff --git a/tool/mkspeedsql.tcl b/tool/mkspeedsql.tcl new file mode 100644 index 0000000..04bafc0 --- /dev/null +++ b/tool/mkspeedsql.tcl @@ -0,0 +1,237 @@ +# 2008 October 9 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file generates SQL text used for performance testing. +# +# $Id: mkspeedsql.tcl,v 1.1 2008/10/09 17:57:34 drh Exp $ +# + +# Set a uniform random seed +expr srand(0) + +# The number_name procedure below converts its argment (an integer) +# into a string which is the English-language name for that number. +# +# Example: +# +# puts [number_name 123] -> "one hundred twenty three" +# +set ones {zero one two three four five six seven eight nine + ten eleven twelve thirteen fourteen fifteen sixteen seventeen + eighteen nineteen} +set tens {{} ten twenty thirty forty fifty sixty seventy eighty ninety} +proc number_name {n} { + if {$n>=1000} { + set txt "[number_name [expr {$n/1000}]] thousand" + set n [expr {$n%1000}] + } else { + set txt {} + } + if {$n>=100} { + append txt " [lindex $::ones [expr {$n/100}]] hundred" + set n [expr {$n%100}] + } + if {$n>=20} { + append txt " [lindex $::tens [expr {$n/10}]]" + set n [expr {$n%10}] + } + if {$n>0} { + append txt " [lindex $::ones $n]" + } + set txt [string trim $txt] + if {$txt==""} {set txt zero} + return $txt +} + +# Create a database schema. +# +puts { + PRAGMA page_size=1024; + PRAGMA cache_size=8192; + PRAGMA locking_mode=EXCLUSIVE; + CREATE TABLE t1(a INTEGER, b INTEGER, c TEXT); + CREATE TABLE t2(a INTEGER, b INTEGER, c TEXT); + CREATE INDEX i2a ON t2(a); + CREATE INDEX i2b ON t2(b); + SELECT name FROM sqlite_master ORDER BY 1; +} + + +# 50000 INSERTs on an unindexed table +# +set t1c_list {} +puts {BEGIN;} +for {set i 1} {$i<=50000} {incr i} { + set r [expr {int(rand()*500000)}] + set x [number_name $r] + lappend t1c_list $x + puts "INSERT INTO t1 VALUES($i,$r,'$x');" +} +puts {COMMIT;} + +# 50000 INSERTs on an indexed table +# +puts {BEGIN;} +for {set i 1} {$i<=50000} {incr i} { + set r [expr {int(rand()*500000)}] + puts "INSERT INTO t2 VALUES($i,$r,'[number_name $r]');" +} +puts {COMMIT;} + + +# 50 SELECTs on an integer comparison. There is no index so +# a full table scan is required. +# +for {set i 0} {$i<50} {incr i} { + set lwr [expr {$i*100}] + set upr [expr {($i+10)*100}] + puts "SELECT count(*), avg(b) FROM t1 WHERE b>=$lwr AND b<$upr;" +} + +# 50 SELECTs on an LIKE comparison. There is no index so a full +# table scan is required. +# +for {set i 0} {$i<50} {incr i} { + puts "SELECT count(*), avg(b) FROM t1 WHERE c LIKE '%[number_name $i]%';" +} + +# Create indices +# +puts {BEGIN;} +puts { + CREATE INDEX i1a ON t1(a); + CREATE INDEX i1b ON t1(b); + CREATE INDEX i1c ON t1(c); +} +puts {COMMIT;} + +# 5000 SELECTs on an integer comparison where the integer is +# indexed. +# +set sql {} +for {set i 0} {$i<5000} {incr i} { + set lwr [expr {$i*100}] + set upr [expr {($i+10)*100}] + puts "SELECT count(*), avg(b) FROM t1 WHERE b>=$lwr AND b<$upr;" +} + +# 100000 random SELECTs against rowid. +# +for {set i 1} {$i<=100000} {incr i} { + set id [expr {int(rand()*50000)+1}] + puts "SELECT c FROM t1 WHERE rowid=$id;" +} + +# 100000 random SELECTs against a unique indexed column. +# +for {set i 1} {$i<=100000} {incr i} { + set id [expr {int(rand()*50000)+1}] + puts "SELECT c FROM t1 WHERE a=$id;" +} + +# 50000 random SELECTs against an indexed column text column +# +set nt1c [llength $t1c_list] +for {set i 0} {$i<50000} {incr i} { + set r [expr {int(rand()*$nt1c)}] + set c [lindex $t1c_list $i] + puts "SELECT c FROM t1 WHERE c='$c';" +} + + +# Vacuum +puts {VACUUM;} + +# 5000 updates of ranges where the field being compared is indexed. +# +puts {BEGIN;} +for {set i 0} {$i<5000} {incr i} { + set lwr [expr {$i*2}] + set upr [expr {($i+1)*2}] + puts "UPDATE t1 SET b=b*2 WHERE a>=$lwr AND a<$upr;" +} +puts {COMMIT;} + +# 50000 single-row updates. An index is used to find the row quickly. +# +puts {BEGIN;} +for {set i 0} {$i<50000} {incr i} { + set r [expr {int(rand()*500000)}] + puts "UPDATE t1 SET b=$r WHERE a=$i;" +} +puts {COMMIT;} + +# 1 big text update that touches every row in the table. +# +puts { + UPDATE t1 SET c=a; +} + +# Many individual text updates. Each row in the table is +# touched through an index. +# +puts {BEGIN;} +for {set i 1} {$i<=50000} {incr i} { + set r [expr {int(rand()*500000)}] + puts "UPDATE t1 SET c='[number_name $r]' WHERE a=$i;" +} +puts {COMMIT;} + +# Delete all content in a table. +# +puts {DELETE FROM t1;} + +# Copy one table into another +# +puts {INSERT INTO t1 SELECT * FROM t2;} + +# Delete all content in a table, one row at a time. +# +puts {DELETE FROM t1 WHERE 1;} + +# Refill the table yet again +# +puts {INSERT INTO t1 SELECT * FROM t2;} + +# Drop the table and recreate it without its indices. +# +puts {BEGIN;} +puts { + DROP TABLE t1; + CREATE TABLE t1(a INTEGER, b INTEGER, c TEXT); +} +puts {COMMIT;} + +# Refill the table yet again. This copy should be faster because +# there are no indices to deal with. +# +puts {INSERT INTO t1 SELECT * FROM t2;} + +# Select 20000 rows from the table at random. +# +puts { + SELECT rowid FROM t1 ORDER BY random() LIMIT 20000; +} + +# Delete 20000 random rows from the table. +# +puts { + DELETE FROM t1 WHERE rowid IN + (SELECT rowid FROM t1 ORDER BY random() LIMIT 20000); +} +puts {SELECT count(*) FROM t1;} + +# Delete 20000 more rows at random from the table. +# +puts { + DELETE FROM t1 WHERE rowid IN + (SELECT rowid FROM t1 ORDER BY random() LIMIT 20000); +} +puts {SELECT count(*) FROM t1;} diff --git a/tool/mksqlite3c-noext.tcl b/tool/mksqlite3c-noext.tcl new file mode 100644 index 0000000..8452072 --- /dev/null +++ b/tool/mksqlite3c-noext.tcl @@ -0,0 +1,367 @@ +#!/usr/bin/tclsh +# +# To build a single huge source file holding all of SQLite (or at +# least the core components - the test harness, shell, and TCL +# interface are omitted.) first do +# +# make target_source +# +# The make target above moves all of the source code files into +# a subdirectory named "tsrc". (This script expects to find the files +# there and will not work if they are not found.) There are a few +# generated C code files that are also added to the tsrc directory. +# For example, the "parse.c" and "parse.h" files to implement the +# the parser are derived from "parse.y" using lemon. And the +# "keywordhash.h" files is generated by a program named "mkkeywordhash". +# +# After the "tsrc" directory has been created and populated, run +# this script: +# +# tclsh mksqlite3c-noext.tcl +# +# The amalgamated SQLite code will be written into sqlite3.c +# + +# Begin by reading the "sqlite3.h" header file. Extract the version number +# from in this file. The version number is needed to generate the header +# comment of the amalgamation. +# +set addstatic 1 +set linemacros 0 +set useapicall 0 +for {set i 0} {$i<[llength $argv]} {incr i} { + set x [lindex $argv $i] + if {[regexp {^-+nostatic$} $x]} { + set addstatic 0 + } elseif {[regexp {^-+linemacros} $x]} { + set linemacros 1 + } elseif {[regexp {^-+useapicall} $x]} { + set useapicall 1 + } else { + error "unknown command-line option: $x" + } +} +set in [open tsrc/sqlite3.h] +set cnt 0 +set VERSION ????? +while {![eof $in]} { + set line [gets $in] + if {$line=="" && [eof $in]} break + incr cnt + regexp {#define\s+SQLITE_VERSION\s+"(.*)"} $line all VERSION +} +close $in + +# Open the output file and write a header comment at the beginning +# of the file. +# +set out [open sqlite3.c w] +# Force the output to use unix line endings, even on Windows. +fconfigure $out -translation lf +set today [clock format [clock seconds] -format "%Y-%m-%d %H:%M:%S UTC" -gmt 1] +puts $out [subst \ +{/****************************************************************************** +** This file is an amalgamation of many separate C source files from SQLite +** version $VERSION. By combining all the individual C code files into this +** single large file, the entire code can be compiled as a single translation +** unit. This allows many compilers to do optimizations that would not be +** possible if the files were compiled separately. Performance improvements +** of 5% or more are commonly seen when SQLite is compiled as a single +** translation unit. +** +** This file is all you need to compile SQLite. To use SQLite in other +** programs, you need this file and the "sqlite3.h" header file that defines +** the programming interface to the SQLite library. (If you do not have +** the "sqlite3.h" header file at hand, you will find a copy embedded within +** the text of this file. Search for "Begin file sqlite3.h" to find the start +** of the embedded sqlite3.h header file.) Additional code files may be needed +** if you want a wrapper to interface SQLite with your choice of programming +** language. The code for the "sqlite3" command-line shell is also in a +** separate file. This file contains only code for the core SQLite library. +*/ +#define SQLITE_CORE 1 +#define SQLITE_AMALGAMATION 1}] +if {$addstatic} { + puts $out \ +{#ifndef SQLITE_PRIVATE +# define SQLITE_PRIVATE static +#endif} +} + +# These are the header files used by SQLite. The first time any of these +# files are seen in a #include statement in the C code, include the complete +# text of the file in-line. The file only needs to be included once. +# +foreach hdr { + btree.h + btreeInt.h + hash.h + hwtime.h + keywordhash.h + msvc.h + mutex.h + opcodes.h + os_common.h + os_setup.h + os_win.h + os.h + pager.h + parse.h + pcache.h + pragma.h + sqlite3.h + sqlite3ext.h + sqliteicu.h + sqliteInt.h + sqliteLimit.h + vdbe.h + vdbeInt.h + vxworks.h + wal.h + whereInt.h +} { + set available_hdr($hdr) 1 +} +set available_hdr(sqliteInt.h) 0 + +# These headers should be copied into the amalgamation without modifying any +# of their function declarations or definitions. +set varonly_hdr(sqlite3.h) 1 + +# These are the functions that accept a variable number of arguments. They +# always need to use the "cdecl" calling convention even when another calling +# convention (e.g. "stcall") is being used for the rest of the library. +set cdecllist { + sqlite3_config + sqlite3_db_config + sqlite3_log + sqlite3_mprintf + sqlite3_snprintf + sqlite3_test_control + sqlite3_vtab_config +} + +# 78 stars used for comment formatting. +set s78 \ +{*****************************************************************************} + +# Insert a comment into the code +# +proc section_comment {text} { + global out s78 + set n [string length $text] + set nstar [expr {60 - $n}] + set stars [string range $s78 0 $nstar] + puts $out "/************** $text $stars/" +} + +# Read the source file named $filename and write it into the +# sqlite3.c output file. If any #include statements are seen, +# process them appropriately. +# +proc copy_file {filename} { + global seen_hdr available_hdr varonly_hdr cdecllist out + global addstatic linemacros useapicall + set ln 0 + set tail [file tail $filename] + section_comment "Begin file $tail" + if {$linemacros} {puts $out "#line 1 \"$filename\""} + set in [open $filename r] + set varpattern {^[a-zA-Z][a-zA-Z_0-9 *]+(sqlite3[_a-zA-Z0-9]+)(\[|;| =)} + set declpattern {([a-zA-Z][a-zA-Z_0-9 ]+ \**)(sqlite3[_a-zA-Z0-9]+)(\(.*)} + if {[file extension $filename]==".h"} { + set declpattern " *$declpattern" + } + set declpattern ^$declpattern\$ + while {![eof $in]} { + set line [gets $in] + incr ln + if {[regexp {^\s*#\s*include\s+["<]([^">]+)[">]} $line all hdr]} { + if {[info exists available_hdr($hdr)]} { + if {$available_hdr($hdr)} { + if {$hdr!="os_common.h" && $hdr!="hwtime.h"} { + set available_hdr($hdr) 0 + } + section_comment "Include $hdr in the middle of $tail" + copy_file tsrc/$hdr + section_comment "Continuing where we left off in $tail" + if {$linemacros} {puts $out "#line [expr {$ln+1}] \"$filename\""} + } else { + # Comment out the entire line, replacing any nested comment + # begin/end markers with the harmless substring "**". + puts $out "/* [string map [list /* ** */ **] $line] */" + } + } elseif {![info exists seen_hdr($hdr)]} { + if {![regexp {/\*\s+amalgamator:\s+dontcache\s+\*/} $line]} { + set seen_hdr($hdr) 1 + } + puts $out $line + } elseif {[regexp {/\*\s+amalgamator:\s+keep\s+\*/} $line]} { + # This include file must be kept because there was a "keep" + # directive inside of a line comment. + puts $out $line + } else { + # Comment out the entire line, replacing any nested comment + # begin/end markers with the harmless substring "**". + puts $out "/* [string map [list /* ** */ **] $line] */" + } + } elseif {[regexp {^#ifdef __cplusplus} $line]} { + puts $out "#if 0" + } elseif {!$linemacros && [regexp {^#line} $line]} { + # Skip #line directives. + } elseif {$addstatic + && ![regexp {^(static|typedef|SQLITE_PRIVATE)} $line]} { + # Skip adding the SQLITE_PRIVATE or SQLITE_API keyword before + # functions if this header file does not need it. + if {![info exists varonly_hdr($tail)] + && [regexp $declpattern $line all rettype funcname rest]} { + regsub {^SQLITE_API } $line {} line + # Add the SQLITE_PRIVATE or SQLITE_API keyword before functions. + # so that linkage can be modified at compile-time. + if {[regexp {^sqlite3[a-z]*_} $funcname]} { + set line SQLITE_API + append line " " [string trim $rettype] + if {[string index $rettype end] ne "*"} { + append line " " + } + if {$useapicall} { + if {[lsearch -exact $cdecllist $funcname] >= 0} { + append line SQLITE_CDECL " " + } else { + append line SQLITE_APICALL " " + } + } + append line $funcname $rest + puts $out $line + } else { + puts $out "SQLITE_PRIVATE $line" + } + } elseif {[regexp $varpattern $line all varname]} { + # Add the SQLITE_PRIVATE before variable declarations or + # definitions for internal use + regsub {^SQLITE_API } $line {} line + if {![regexp {^sqlite3_} $varname]} { + regsub {^extern } $line {} line + puts $out "SQLITE_PRIVATE $line" + } else { + if {[regexp {const char sqlite3_version\[\];} $line]} { + set line {const char sqlite3_version[] = SQLITE_VERSION;} + } + regsub {^SQLITE_EXTERN } $line {} line + puts $out "SQLITE_API $line" + } + } elseif {[regexp {^(SQLITE_EXTERN )?void \(\*sqlite3IoTrace\)} $line]} { + regsub {^SQLITE_API } $line {} line + regsub {^SQLITE_EXTERN } $line {} line + puts $out $line + } elseif {[regexp {^void \(\*sqlite3Os} $line]} { + regsub {^SQLITE_API } $line {} line + puts $out "SQLITE_PRIVATE $line" + } else { + puts $out $line + } + } else { + puts $out $line + } + } + close $in + section_comment "End of $tail" +} + + +# Process the source files. Process files containing commonly +# used subroutines first in order to help the compiler find +# inlining opportunities. +# +foreach file { + sqliteInt.h + + global.c + ctime.c + status.c + date.c + os.c + + fault.c + mem0.c + mem1.c + mem2.c + mem3.c + mem5.c + mutex.c + mutex_noop.c + mutex_unix.c + mutex_w32.c + malloc.c + printf.c + treeview.c + random.c + threads.c + utf.c + util.c + hash.c + opcodes.c + + os_unix.c + os_win.c + + bitvec.c + pcache.c + pcache1.c + rowset.c + pager.c + wal.c + + btmutex.c + btree.c + backup.c + + vdbemem.c + vdbeaux.c + vdbeapi.c + vdbetrace.c + vdbe.c + vdbeblob.c + vdbesort.c + memjournal.c + + walker.c + resolve.c + expr.c + alter.c + analyze.c + attach.c + auth.c + build.c + callback.c + delete.c + func.c + fkey.c + insert.c + legacy.c + loadext.c + pragma.c + prepare.c + select.c + table.c + trigger.c + update.c + vacuum.c + vtab.c + wherecode.c + whereexpr.c + where.c + window.c + + parse.c + + tokenize.c + complete.c + + main.c + notify.c +} { + copy_file tsrc/$file +} + +close $out diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl new file mode 100644 index 0000000..53fa59a --- /dev/null +++ b/tool/mksqlite3c.tcl @@ -0,0 +1,488 @@ +#!/usr/bin/tclsh +# +# To build a single huge source file holding all of SQLite (or at +# least the core components - the test harness, shell, and TCL +# interface are omitted.) first do +# +# make target_source +# +# The make target above moves all of the source code files into +# a subdirectory named "tsrc". (This script expects to find the files +# there and will not work if they are not found.) There are a few +# generated C code files that are also added to the tsrc directory. +# For example, the "parse.c" and "parse.h" files to implement the +# the parser are derived from "parse.y" using lemon. And the +# "keywordhash.h" files is generated by a program named "mkkeywordhash". +# +# After the "tsrc" directory has been created and populated, run +# this script: +# +# tclsh mksqlite3c.tcl +# +# The amalgamated SQLite code will be written into sqlite3.c +# + +set help {Usage: tclsh mksqlite3c.tcl <options> + where <options> is zero or more of the following with these effects: + --nostatic => Do not generate with compile-time modifiable linkage. + --linemacros=? => Emit #line directives into output or not. (? = 1 or 0) + --useapicall => Prepend functions with SQLITE_APICALL or SQLITE_CDECL. + --srcdir $SRC => Specify the directory containing constituent sources. + --help => See this. + The value setting options default to --linemacros=1 and '--srcdir tsrc' . +} + +# Begin by reading the "sqlite3.h" header file. Extract the version number +# from in this file. The version number is needed to generate the header +# comment of the amalgamation. +# + +set addstatic 1 +set linemacros 0 +set useapicall 0 +set enable_recover 0 +set srcdir tsrc + +for {set i 0} {$i<[llength $argv]} {incr i} { + set x [lindex $argv $i] + if {[regexp {^-?-enable-recover$} $x]} { + set enable_recover 1 + } elseif {[regexp {^-?-nostatic$} $x]} { + set addstatic 0 + } elseif {[regexp {^-?-linemacros(?:=([01]))?$} $x ma ulm]} { + if {$ulm == ""} {set ulm 1} + set linemacros $ulm + } elseif {[regexp {^-?-useapicall$} $x]} { + set useapicall 1 + } elseif {[regexp {^-?-srcdir$} $x]} { + incr i + if {$i==[llength $argv]} { + error "No argument following $x" + } + set srcdir [lindex $argv $i] + } elseif {[regexp {^-?-((help)|\?)$} $x]} { + puts $help + exit 0 + } else { + error "unknown command-line option: $x" + } +} +set in [open $srcdir/sqlite3.h] +set cnt 0 +set VERSION ????? +while {![eof $in]} { + set line [gets $in] + if {$line=="" && [eof $in]} break + incr cnt + regexp {#define\s+SQLITE_VERSION\s+"(.*)"} $line all VERSION +} +close $in + +# Open the output file and write a header comment at the beginning +# of the file. +# +set fname sqlite3.c +if {$enable_recover} { set fname sqlite3r.c } +set out [open $fname w] +# Force the output to use unix line endings, even on Windows. +fconfigure $out -translation lf +set today [clock format [clock seconds] -format "%Y-%m-%d %H:%M:%S UTC" -gmt 1] +puts $out [subst \ +{/****************************************************************************** +** This file is an amalgamation of many separate C source files from SQLite +** version $VERSION. By combining all the individual C code files into this +** single large file, the entire code can be compiled as a single translation +** unit. This allows many compilers to do optimizations that would not be +** possible if the files were compiled separately. Performance improvements +** of 5% or more are commonly seen when SQLite is compiled as a single +** translation unit. +** +** This file is all you need to compile SQLite. To use SQLite in other +** programs, you need this file and the "sqlite3.h" header file that defines +** the programming interface to the SQLite library. (If you do not have +** the "sqlite3.h" header file at hand, you will find a copy embedded within +** the text of this file. Search for "Begin file sqlite3.h" to find the start +** of the embedded sqlite3.h header file.) Additional code files may be needed +** if you want a wrapper to interface SQLite with your choice of programming +** language. The code for the "sqlite3" command-line shell is also in a +** separate file. This file contains only code for the core SQLite library. +**}] +set srcroot [file dirname [file dirname [info script]]] +if {$tcl_platform(platform)=="windows"} { + set vsrcprog src-verify.exe +} else { + set vsrcprog ./src-verify +} +if {[file executable $vsrcprog] && [file readable $srcroot/manifest]} { + set res [string trim [split [exec $vsrcprog -x $srcroot]] \n] + puts $out "** The content in this amalgamation comes from Fossil check-in" + puts -nonewline $out "** [string range [lindex $res 0] 0 35]" + if {[llength $res]==1} { + puts $out "." + } else { + puts $out " with changes in files:\n**" + foreach f [lrange $res 1 end] { + puts $out "** $f" + } + } +} else { + puts $out "** The origin of the sources used to build this amalgamation" + puts $out "** is unknown." +} +puts $out [subst {*/ +#define SQLITE_CORE 1 +#define SQLITE_AMALGAMATION 1}] +if {$addstatic} { + puts $out \ +{#ifndef SQLITE_PRIVATE +# define SQLITE_PRIVATE static +#endif} +} + +# Examine the parse.c file. If it contains lines of the form: +# +# "#ifndef SQLITE_ENABLE_UPDATE_LIMIT +# +# then set the SQLITE_UDL_CAPABLE_PARSER flag in the amalgamation. +# +set in [open $srcdir/parse.c] +if {[regexp {ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT} [read $in]]} { + puts $out "#define SQLITE_UDL_CAPABLE_PARSER 1" +} +close $in + +# These are the header files used by SQLite. The first time any of these +# files are seen in a #include statement in the C code, include the complete +# text of the file in-line. The file only needs to be included once. +# +foreach hdr { + btree.h + btreeInt.h + fts3.h + fts3Int.h + fts3_hash.h + fts3_tokenizer.h + geopoly.c + hash.h + hwtime.h + keywordhash.h + msvc.h + mutex.h + opcodes.h + os_common.h + os_setup.h + os_win.h + os.h + pager.h + parse.h + pcache.h + pragma.h + rtree.h + sqlite3session.h + sqlite3.h + sqlite3ext.h + sqlite3rbu.h + sqliteicu.h + sqliteInt.h + sqliteLimit.h + vdbe.h + vdbeInt.h + vxworks.h + wal.h + whereInt.h + sqlite3recover.h +} { + set available_hdr($hdr) 1 +} +set available_hdr(sqliteInt.h) 0 +set available_hdr(os_common.h) 0 +set available_hdr(sqlite3session.h) 0 + +# These headers should be copied into the amalgamation without modifying any +# of their function declarations or definitions. +set varonly_hdr(sqlite3.h) 1 + +# These are the functions that accept a variable number of arguments. They +# always need to use the "cdecl" calling convention even when another calling +# convention (e.g. "stcall") is being used for the rest of the library. +set cdecllist { + sqlite3_config + sqlite3_db_config + sqlite3_log + sqlite3_mprintf + sqlite3_snprintf + sqlite3_test_control + sqlite3_vtab_config +} + +# 78 stars used for comment formatting. +set s78 \ +{*****************************************************************************} + +# Insert a comment into the code +# +proc section_comment {text} { + global out s78 + set n [string length $text] + set nstar [expr {60 - $n}] + set stars [string range $s78 0 $nstar] + puts $out "/************** $text $stars/" +} + +# Read the source file named $filename and write it into the +# sqlite3.c output file. If any #include statements are seen, +# process them appropriately. +# +proc copy_file {filename} { + global seen_hdr available_hdr varonly_hdr cdecllist out + global addstatic linemacros useapicall srcdir + set ln 0 + set tail [file tail $filename] + section_comment "Begin file $tail" + if {$linemacros} {puts $out "#line 1 \"$filename\""} + set in [open $filename r] + set varpattern {^[a-zA-Z][a-zA-Z_0-9 *]+(sqlite3[_a-zA-Z0-9]+)(\[|;| =)} + set declpattern {([a-zA-Z][a-zA-Z_0-9 ]+ \**)(sqlite3[_a-zA-Z0-9]+)(\(.*)} + if {[file extension $filename]==".h"} { + set declpattern " *$declpattern" + } + set declpattern ^$declpattern\$ + while {![eof $in]} { + set line [string trimright [gets $in]] + incr ln + if {[regexp {^\s*#\s*include\s+["<]([^">]+)[">]} $line all hdr]} { + if {[info exists available_hdr($hdr)]} { + if {$available_hdr($hdr)} { + set available_hdr($hdr) 0 + section_comment "Include $hdr in the middle of $tail" + copy_file $srcdir/$hdr + section_comment "Continuing where we left off in $tail" + if {$linemacros} {puts $out "#line [expr {$ln+1}] \"$filename\""} + } else { + # Comment out the entire line, replacing any nested comment + # begin/end markers with the harmless substring "**". + puts $out "/* [string map [list /* ** */ **] $line] */" + } + } elseif {![info exists seen_hdr($hdr)]} { + if {![regexp {/\*\s+amalgamator:\s+dontcache\s+\*/} $line]} { + set seen_hdr($hdr) 1 + } + puts $out $line + } elseif {[regexp {/\*\s+amalgamator:\s+keep\s+\*/} $line]} { + # This include file must be kept because there was a "keep" + # directive inside of a line comment. + puts $out $line + } else { + # Comment out the entire line, replacing any nested comment + # begin/end markers with the harmless substring "**". + puts $out "/* [string map [list /* ** */ **] $line] */" + } + } elseif {[regexp {^#ifdef __cplusplus} $line]} { + puts $out "#if 0" + } elseif {!$linemacros && [regexp {^#line} $line]} { + # Skip #line directives. + } elseif {$addstatic + && ![regexp {^(static|typedef|SQLITE_PRIVATE)} $line]} { + # Skip adding the SQLITE_PRIVATE or SQLITE_API keyword before + # functions if this header file does not need it. + if {![info exists varonly_hdr($tail)] + && [regexp $declpattern $line all rettype funcname rest]} { + regsub {^SQLITE_API } $line {} line + regsub {^SQLITE_API } $rettype {} rettype + + # Add the SQLITE_PRIVATE or SQLITE_API keyword before functions. + # so that linkage can be modified at compile-time. + if {[regexp {^sqlite3[a-z]*_} $funcname]} { + set line SQLITE_API + append line " " [string trim $rettype] + if {[string index $rettype end] ne "*"} { + append line " " + } + if {$useapicall} { + if {[lsearch -exact $cdecllist $funcname] >= 0} { + append line SQLITE_CDECL " " + } else { + append line SQLITE_APICALL " " + } + } + append line $funcname $rest + if {$funcname=="sqlite3_sourceid"} { + # The sqlite3_sourceid() routine is synthesized at the end of + # the amalgamation + puts $out "/* $line */" + } else { + puts $out $line + } + } else { + puts $out "SQLITE_PRIVATE $line" + } + } elseif {[regexp $varpattern $line all varname]} { + # Add the SQLITE_PRIVATE before variable declarations or + # definitions for internal use + regsub {^SQLITE_API } $line {} line + if {![regexp {^sqlite3_} $varname] + && ![regexp {^sqlite3Show[A-Z]} $varname]} { + regsub {^extern } $line {} line + puts $out "SQLITE_PRIVATE $line" + } else { + if {[regexp {const char sqlite3_version\[\];} $line]} { + set line {const char sqlite3_version[] = SQLITE_VERSION;} + } + regsub {^SQLITE_EXTERN } $line {} line + puts $out "SQLITE_API $line" + } + } elseif {[regexp {^(SQLITE_EXTERN )?void \(\*sqlite3IoTrace\)} $line]} { + regsub {^SQLITE_API } $line {} line + regsub {^SQLITE_EXTERN } $line {} line + puts $out $line + } elseif {[regexp {^void \(\*sqlite3Os} $line]} { + regsub {^SQLITE_API } $line {} line + puts $out "SQLITE_PRIVATE $line" + } else { + puts $out $line + } + } else { + puts $out $line + } + } + close $in + section_comment "End of $tail" +} + + +# Process the source files. Process files containing commonly +# used subroutines first in order to help the compiler find +# inlining opportunities. +# +set flist { + sqliteInt.h + os_common.h + ctime.c + + global.c + status.c + date.c + os.c + + fault.c + mem0.c + mem1.c + mem2.c + mem3.c + mem5.c + mutex.c + mutex_noop.c + mutex_unix.c + mutex_w32.c + malloc.c + printf.c + treeview.c + random.c + threads.c + utf.c + util.c + hash.c + opcodes.c + + os_kv.c + os_unix.c + os_win.c + memdb.c + + bitvec.c + pcache.c + pcache1.c + rowset.c + pager.c + wal.c + + btmutex.c + btree.c + backup.c + + vdbemem.c + vdbeaux.c + vdbeapi.c + vdbetrace.c + vdbe.c + vdbeblob.c + vdbesort.c + vdbevtab.c + memjournal.c + + walker.c + resolve.c + expr.c + alter.c + analyze.c + attach.c + auth.c + build.c + callback.c + delete.c + func.c + fkey.c + insert.c + legacy.c + loadext.c + pragma.c + prepare.c + select.c + table.c + trigger.c + update.c + upsert.c + vacuum.c + vtab.c + wherecode.c + whereexpr.c + where.c + window.c + + parse.c + + tokenize.c + complete.c + + main.c + notify.c + + fts3.c + fts3_aux.c + fts3_expr.c + fts3_hash.c + fts3_porter.c + fts3_tokenizer.c + fts3_tokenizer1.c + fts3_tokenize_vtab.c + fts3_write.c + fts3_snippet.c + fts3_unicode.c + fts3_unicode2.c + + json.c + rtree.c + icu.c + fts3_icu.c + sqlite3rbu.c + dbstat.c + dbpage.c + sqlite3session.c + fts5.c + stmt.c +} +if {$enable_recover} { + lappend flist sqlite3recover.c dbdata.c +} +foreach file $flist { + copy_file $srcdir/$file +} + +puts $out \ +"/* Return the source-id for this library */ +SQLITE_API const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; }" + +puts $out \ +"/************************** End of sqlite3.c ******************************/" + +close $out diff --git a/tool/mksqlite3h.tcl b/tool/mksqlite3h.tcl new file mode 100644 index 0000000..bd579c2 --- /dev/null +++ b/tool/mksqlite3h.tcl @@ -0,0 +1,165 @@ +#!/usr/bin/tclsh +# +# This script constructs the "sqlite3.h" header file from the following +# sources: +# +# 1) The src/sqlite.h.in source file. This is the template for sqlite3.h. +# 2) The VERSION file containing the current SQLite version number. +# 3) The manifest file from the fossil SCM. This gives use the date. +# 4) The manifest.uuid file from the fossil SCM. This gives the SHA1 hash. +# +# Run this script by specifying the root directory of the source tree +# on the command-line. +# +# This script performs processing on src/sqlite.h.in. It: +# +# 1) Adds SQLITE_EXTERN in front of the declaration of global variables, +# 2) Adds SQLITE_API in front of the declaration of API functions, +# 3) Replaces the string --VERS-- with the current library version, +# formatted as a string (e.g. "3.6.17"), and +# 4) Replaces the string --VERSION-NUMBER-- with current library version, +# formatted as an integer (e.g. "3006017"). +# 5) Replaces the string --SOURCE-ID-- with the date and time and sha1 +# hash of the fossil-scm manifest for the source tree. +# 6) Adds the SQLITE_CALLBACK calling convention macro in front of all +# callback declarations. +# +# This script outputs to stdout. +# +# Example usage: +# +# tclsh mksqlite3h.tcl ../sqlite >sqlite3.h +# + + +# Get the source tree root directory from the command-line +# +set TOP [lindex $argv 0] + +# Enable use of SQLITE_APICALL macros at the right points? +# +set useapicall 0 + +# Include sqlite3recover.h? +# +set enable_recover 0 + +if {[lsearch -regexp [lrange $argv 1 end] {^-+useapicall}] != -1} { + set useapicall 1 +} +if {[lsearch -regexp [lrange $argv 1 end] {^-+enable-recover}] != -1} { + set enable_recover 1 +} + +# Get the SQLite version number (ex: 3.6.18) from the $TOP/VERSION file. +# +set in [open $TOP/VERSION] +set zVersion [string trim [read $in]] +close $in +set nVersion [eval format "%d%03d%03d" [split $zVersion .]] + +# Get the source-id +# +set PWD [pwd] +cd $TOP +set zSourceId [exec $PWD/mksourceid manifest] +cd $PWD + +# Set up patterns for recognizing API declarations. +# +set varpattern {^[a-zA-Z][a-zA-Z_0-9 *]+sqlite3_[_a-zA-Z0-9]+(\[|;| =)} +set declpattern1 {^ *([a-zA-Z][a-zA-Z_0-9 ]+ \**)(sqlite3_[_a-zA-Z0-9]+)(\(.*)$} + +set declpattern2 \ + {^ *([a-zA-Z][a-zA-Z_0-9 ]+ \**)(sqlite3session_[_a-zA-Z0-9]+)(\(.*)$} + +set declpattern3 \ + {^ *([a-zA-Z][a-zA-Z_0-9 ]+ \**)(sqlite3changeset_[_a-zA-Z0-9]+)(\(.*)$} + +set declpattern4 \ + {^ *([a-zA-Z][a-zA-Z_0-9 ]+ \**)(sqlite3changegroup_[_a-zA-Z0-9]+)(\(.*)$} + +set declpattern5 \ + {^ *([a-zA-Z][a-zA-Z_0-9 ]+ \**)(sqlite3rebaser_[_a-zA-Z0-9]+)(\(.*)$} + +# Force the output to use unix line endings, even on Windows. +fconfigure stdout -translation lf + +set filelist [subst { + $TOP/src/sqlite.h.in + $TOP/ext/rtree/sqlite3rtree.h + $TOP/ext/session/sqlite3session.h + $TOP/ext/fts5/fts5.h +}] +if {$enable_recover} { + lappend filelist "$TOP/ext/recover/sqlite3recover.h" +} + +# These are the functions that accept a variable number of arguments. They +# always need to use the "cdecl" calling convention even when another calling +# convention (e.g. "stcall") is being used for the rest of the library. +set cdecllist { + sqlite3_config + sqlite3_db_config + sqlite3_log + sqlite3_mprintf + sqlite3_snprintf + sqlite3_test_control + sqlite3_vtab_config +} + +# Process the source files. +# +foreach file $filelist { + set in [open $file] + if {![regexp {sqlite\.h\.in} $file]} { + puts "/******** Begin file [file tail $file] *********/" + } + while {![eof $in]} { + + set line [string trimright [gets $in]] + + # File sqlite3rtree.h contains a line "#include <sqlite3.h>". Omit this + # line when copying sqlite3rtree.h into sqlite3.h. + # + if {[string match {*#include*[<"]sqlite3.h[>"]*} $line]} continue + + regsub -- --VERS-- $line $zVersion line + regsub -- --VERSION-NUMBER-- $line $nVersion line + regsub -- --SOURCE-ID-- $line "$zSourceId" line + + if {[regexp $varpattern $line] && ![regexp {^ *typedef} $line]} { + set line "SQLITE_API $line" + } else { + if {[regexp $declpattern1 $line all rettype funcname rest] || \ + [regexp $declpattern2 $line all rettype funcname rest] || \ + [regexp $declpattern3 $line all rettype funcname rest] || \ + [regexp $declpattern4 $line all rettype funcname rest] || \ + [regexp $declpattern5 $line all rettype funcname rest]} { + set line SQLITE_API + append line " " [string trim $rettype] + if {[string index $rettype end] ne "*"} { + append line " " + } + if {$useapicall} { + if {[lsearch -exact $cdecllist $funcname] >= 0} { + append line SQLITE_CDECL " " + } else { + append line SQLITE_APICALL " " + } + } + append line $funcname $rest + } + } + if {$useapicall} { + set line [string map [list (*sqlite3_syscall_ptr) \ + "(SQLITE_SYSAPI *sqlite3_syscall_ptr)"] $line] + regsub {\(\*} $line {(SQLITE_CALLBACK *} line + } + puts $line + } + close $in + if {![regexp {sqlite\.h\.in} $file]} { + puts "/******** End of [file tail $file] *********/" + } +} diff --git a/tool/mksqlite3internalh.tcl b/tool/mksqlite3internalh.tcl new file mode 100644 index 0000000..8db593f --- /dev/null +++ b/tool/mksqlite3internalh.tcl @@ -0,0 +1,148 @@ +#!/usr/bin/tclsh +# +# To build a single huge source file holding all of SQLite (or at +# least the core components - the test harness, shell, and TCL +# interface are omitted.) first do +# +# make target_source +# +# The make target above moves all of the source code files into +# a subdirectory named "tsrc". (This script expects to find the files +# there and will not work if they are not found.) There are a few +# generated C code files that are also added to the tsrc directory. +# For example, the "parse.c" and "parse.h" files to implement the +# the parser are derived from "parse.y" using lemon. And the +# "keywordhash.h" files is generated by a program named "mkkeywordhash". +# +# After the "tsrc" directory has been created and populated, run +# this script: +# +# tclsh mksqlite3c.tcl +# +# The amalgamated SQLite code will be written into sqlite3.c +# + +# Begin by reading the "sqlite3.h" header file. Count the number of lines +# in this file and extract the version number. That information will be +# needed in order to generate the header of the amalgamation. +# +set in [open tsrc/sqlite3.h] +set cnt 0 +set VERSION ????? +while {![eof $in]} { + set line [gets $in] + if {$line=="" && [eof $in]} break + incr cnt + regexp {#define\s+SQLITE_VERSION\s+"(.*)"} $line all VERSION +} +close $in + +# Open the output file and write a header comment at the beginning +# of the file. +# +set out [open sqlite3internal.h w] +set today [clock format [clock seconds] -format "%Y-%m-%d %H:%M:%S UTC" -gmt 1] +puts $out [subst \ +{/****************************************************************************** +** This file is an amalgamation of many private header files from SQLite +** version $VERSION. +*/}] + +# These are the header files used by SQLite. The first time any of these +# files are seen in a #include statement in the C code, include the complete +# text of the file in-line. The file only needs to be included once. +# +foreach hdr { + btree.h + btreeInt.h + hash.h + hwtime.h + keywordhash.h + msvc.h + opcodes.h + os_common.h + os_setup.h + os_win.h + os.h + pager.h + parse.h + sqlite3ext.h + sqlite3.h + sqliteInt.h + sqliteLimit.h + vdbe.h + vdbeInt.h +} { + set available_hdr($hdr) 1 +} + +# 78 stars used for comment formatting. +set s78 \ +{*****************************************************************************} + +# Insert a comment into the code +# +proc section_comment {text} { + global out s78 + set n [string length $text] + set nstar [expr {60 - $n}] + set stars [string range $s78 0 $nstar] + puts $out "/************** $text $stars/" +} + +# Read the source file named $filename and write it into the +# sqlite3.c output file. If any #include statements are seen, +# process them approprately. +# +proc copy_file {filename} { + global seen_hdr available_hdr out + set tail [file tail $filename] + section_comment "Begin file $tail" + set in [open $filename r] + while {![eof $in]} { + set line [gets $in] + if {[regexp {^#\s*include\s+["<]([^">]+)[">]} $line all hdr]} { + if {[info exists available_hdr($hdr)]} { + if {$available_hdr($hdr)} { + section_comment "Include $hdr in the middle of $tail" + copy_file tsrc/$hdr + section_comment "Continuing where we left off in $tail" + } + } elseif {![info exists seen_hdr($hdr)]} { + set seen_hdr($hdr) 1 + puts $out $line + } + } elseif {[regexp {^#ifdef __cplusplus} $line]} { + puts $out "#if 0" + } elseif {[regexp {^#line} $line]} { + # Skip #line directives. + } else { + puts $out $line + } + } + close $in + section_comment "End of $tail" +} + + +# Process the source files. Process files containing commonly +# used subroutines first in order to help the compiler find +# inlining opportunities. +# +foreach file { + sqliteInt.h + sqlite3.h + btree.h + hash.h + os.h + pager.h + parse.h + sqlite3ext.h + vdbe.h +} { + if {$available_hdr($file)} { + copy_file tsrc/$file + } +} + +close $out diff --git a/tool/mktoolzip.tcl b/tool/mktoolzip.tcl new file mode 100644 index 0000000..885bae9 --- /dev/null +++ b/tool/mktoolzip.tcl @@ -0,0 +1,69 @@ +#!/usr/bin/tclsh +# +# Run this script in order to generate a ZIP archive containing various +# command-line tools. +# +# The makefile that invokes this script must first build the following +# binaries: +# +# testfixture -- used to run this script +# sqlite3 -- the SQLite CLI +# sqldiff -- Program to diff two databases +# sqlite3_analyzer -- Space analyzer +# +switch $tcl_platform(os) { + {Windows NT} { + set OS win32 + set EXE .exe + } + Linux { + set OS linux + set EXE {} + } + Darwin { + set OS osx + set EXE {} + } + default { + set OS unknown + set EXE {} + } +} +switch $tcl_platform(machine) { + arm64 { + set ARCH arm64 + } + x86_64 { + set ARCH x64 + } + amd64 - + intel { + if {$tcl_platform(pointerSize)==4} { + set ARCH x86 + } else { + set ARCH x64 + } + } + default { + set ARCH unk + } +} +set in [open [file join [file dirname [file dirname [info script]]] VERSION]] +set vers [read $in] +close $in +scan $vers %d.%d.%d v1 v2 v3 +set v2 [format 3%02d%02d00 $v2 $v3] +set name sqlite-tools-$OS-$ARCH-$v2.zip + +if {$OS=="win32"} { + # The win32 tar.exe supports the -a ("auto-compress") option. This causes + # tar to create an archive type based on the extension of the output file. + # In this case, a zip file. + puts "tar -a -cf $name sqlite3$EXE sqldiff$EXE sqlite3_analyzer$EXE" + puts [exec tar -a -cf $name sqlite3$EXE sqldiff$EXE sqlite3_analyzer$EXE] + puts "$name: [file size $name] bytes" +} else { + puts "zip $name sqlite3$EXE sqldiff$EXE sqlite3_analyzer$EXE" + puts [exec zip $name sqlite3$EXE sqldiff$EXE sqlite3_analyzer$EXE] + puts [exec ls -l $name] +} diff --git a/tool/mkvsix.tcl b/tool/mkvsix.tcl new file mode 100644 index 0000000..c874d53 --- /dev/null +++ b/tool/mkvsix.tcl @@ -0,0 +1,840 @@ +#!/usr/bin/tclsh +# +# This script is used to generate a VSIX (Visual Studio Extension) file for +# SQLite usable by Visual Studio. +# +# PREREQUISITES +# +# 1. Tcl 8.4 and later are supported, earlier versions have not been tested. +# +# 2. The "sqlite3.h" file is assumed to exist in the parent directory of the +# directory containing this script. The [optional] second command line +# argument to this script may be used to specify an alternate location. +# This script also assumes that the "sqlite3.h" file corresponds with the +# version of the binaries to be packaged. This assumption is not verified +# by this script. +# +# 3. The temporary directory specified in the TEMP or TMP environment variables +# must refer to an existing directory writable by the current user. +# +# 4. The "zip" and "unzip" command line tools must be located either in a +# directory contained in the PATH environment variable or specified as the +# exact file names to execute in the "ZipTool" and "UnZipTool" environment +# variables, respectively. +# +# 5. The template VSIX file (which is basically a zip file) must be located in +# a "win" directory inside the directory containing this script. It should +# not contain any executable binaries. It should only contain dynamic +# textual content files to be processed using [subst] and/or static content +# files to be copied verbatim. +# +# 6. The executable and other compiled binary files to be packaged into the +# final VSIX file (e.g. DLLs, LIBs, and PDBs) must be located in a single +# directory tree. The top-level directory of the tree must be specified as +# the first command line argument to this script. The second level +# sub-directory names must match those of the build configuration (e.g. +# "Debug" or "Retail"). The third level sub-directory names must match +# those of the platform (e.g. "x86", "x64", and "ARM"). For example, the +# binary files to be packaged would need to be organized as follows when +# packaging the "Debug" and "Retail" build configurations for the "x86" and +# "x64" platforms (in this example, "C:\temp" is the top-level directory as +# specified in the first command line argument): +# +# C:\Temp\Debug\x86\sqlite3.lib +# C:\Temp\Debug\x86\sqlite3.dll +# C:\Temp\Debug\x86\sqlite3.pdb +# C:\Temp\Debug\x64\sqlite3.lib +# C:\Temp\Debug\x64\sqlite3.dll +# C:\Temp\Debug\x64\sqlite3.pdb +# C:\Temp\Retail\x86\sqlite3.lib +# C:\Temp\Retail\x86\sqlite3.dll +# C:\Temp\Retail\x86\sqlite3.pdb +# C:\Temp\Retail\x64\sqlite3.lib +# C:\Temp\Retail\x64\sqlite3.dll +# C:\Temp\Retail\x64\sqlite3.pdb +# +# The above directory tree organization is performed automatically if the +# "tool\build-all-msvc.bat" batch script is used to build the binary files +# to be packaged. +# +# USAGE +# +# The first argument to this script is required and must be the name of the +# top-level directory containing the directories and files organized into a +# tree as described in item 6 of the PREREQUISITES section, above. The second +# argument is optional and if present must contain the name of the directory +# containing the root of the source tree for SQLite. The third argument is +# optional and if present must contain the flavor the VSIX package to build. +# Currently, the only supported package flavors are "WinRT", "WinRT81", "WP80", +# "WP81", and "Win32". The fourth argument is optional and if present must be +# a string containing a list of platforms to include in the VSIX package. The +# platform list is "platform1,platform2,platform3". The fifth argument is +# optional and if present must contain the version of Visual Studio required by +# the package. Currently, the only supported versions are "2012" and "2013". +# The package flavors "WinRT81" and "WP81" are only supported when the Visual +# Studio version is "2013". Typically, when on Windows, this script is +# executed using commands similar to the following from a normal Windows +# command prompt: +# +# CD /D C:\dev\sqlite\core +# tclsh tool\mkvsix.tcl C:\Temp +# +# In the example above, "C:\dev\sqlite\core" represents the root of the source +# tree for SQLite and "C:\Temp" represents the top-level directory containing +# the executable and other compiled binary files, organized into a directory +# tree as described in item 6 of the PREREQUISITES section, above. +# +# This script should work on non-Windows platforms as well, provided that all +# the requirements listed in the PREREQUISITES section are met. +# +# NOTES +# +# The temporary directory is used as a staging area for the final VSIX file. +# The template VSIX file is extracted, its contents processed, and then the +# resulting files are packaged into the final VSIX file. +# +package require Tcl 8.4 + +proc fail { {error ""} {usage false} } { + if {[string length $error] > 0} then { + puts stdout $error + if {!$usage} then {exit 1} + } + + puts stdout "usage:\ +[file tail [info nameofexecutable]]\ +[file tail [info script]] <binaryDirectory> \[sourceDirectory\]\ +\[packageFlavor\] \[platformNames\] \[vsVersion\]" + + exit 1 +} + +proc getEnvironmentVariable { name } { + # + # NOTE: Returns the value of the specified environment variable or an empty + # string for environment variables that do not exist in the current + # process environment. + # + return [expr {[info exists ::env($name)] ? $::env($name) : ""}] +} + +proc getTemporaryPath {} { + # + # NOTE: Returns the normalized path to the first temporary directory found + # in the typical set of environment variables used for that purpose + # or an empty string to signal a failure to locate such a directory. + # + set names [list] + + foreach name [list TEMP TMP] { + lappend names [string toupper $name] [string tolower $name] \ + [string totitle $name] + } + + foreach name $names { + set value [getEnvironmentVariable $name] + + if {[string length $value] > 0} then { + return [file normalize $value] + } + } + + return "" +} + +proc appendArgs { args } { + # + # NOTE: Returns all passed arguments joined together as a single string with + # no intervening spaces between arguments. + # + eval append result $args +} + +proc readFile { fileName } { + # + # NOTE: Reads and returns the entire contents of the specified file, which + # may contain binary data. + # + set file_id [open $fileName RDONLY] + fconfigure $file_id -encoding binary -translation binary + set result [read $file_id] + close $file_id + return $result +} + +proc writeFile { fileName data } { + # + # NOTE: Writes the entire contents of the specified file, which may contain + # binary data. + # + set file_id [open $fileName {WRONLY CREAT TRUNC}] + fconfigure $file_id -encoding binary -translation binary + puts -nonewline $file_id $data + close $file_id + return "" +} + +# +# TODO: Modify this procedure when a new version of Visual Studio is released. +# +proc getMinVsVersionXmlChunk { vsVersion } { + switch -exact $vsVersion { + 2012 { + return [appendArgs \ + "\r\n " {MinVSVersion="11.0"}] + } + 2013 { + return [appendArgs \ + "\r\n " {MinVSVersion="12.0"}] + } + 2015 { + return [appendArgs \ + "\r\n " {MinVSVersion="14.0"}] + } + default { + return "" + } + } +} + +# +# TODO: Modify this procedure when a new version of Visual Studio is released. +# +proc getMaxPlatformVersionXmlChunk { packageFlavor vsVersion } { + # + # NOTE: Only Visual Studio 2013 and later support this attribute within the + # SDK manifest. + # + if {![string equal $vsVersion 2013] && \ + ![string equal $vsVersion 2015]} then { + return "" + } + + switch -exact $packageFlavor { + WinRT { + return [appendArgs \ + "\r\n " {MaxPlatformVersion="8.0"}] + } + WinRT81 { + return [appendArgs \ + "\r\n " {MaxPlatformVersion="8.1"}] + } + WP80 { + return [appendArgs \ + "\r\n " {MaxPlatformVersion="8.0"}] + } + WP81 { + return [appendArgs \ + "\r\n " {MaxPlatformVersion="8.1"}] + } + default { + return "" + } + } +} + +# +# TODO: Modify this procedure when a new version of Visual Studio is released. +# +proc getExtraFileListXmlChunk { packageFlavor vsVersion } { + # + # NOTE: Windows Phone 8.0 does not require any extra attributes in its VSIX + # package SDK manifests; however, it appears that Windows Phone 8.1 + # does. + # + if {[string equal $packageFlavor WP80]} then { + return "" + } + + set appliesTo [expr {[string equal $packageFlavor Win32] ? \ + "VisualC" : "WindowsAppContainer"}] + + switch -exact $vsVersion { + 2012 { + return [appendArgs \ + "\r\n " AppliesTo=\" $appliesTo \" \ + "\r\n " {DependsOn="Microsoft.VCLibs, version=11.0"}] + } + 2013 { + return [appendArgs \ + "\r\n " AppliesTo=\" $appliesTo \" \ + "\r\n " {DependsOn="Microsoft.VCLibs, version=12.0"}] + } + 2015 { + return [appendArgs \ + "\r\n " AppliesTo=\" $appliesTo \" \ + "\r\n " {DependsOn="Microsoft.VCLibs, version=14.0"}] + } + default { + return "" + } + } +} + +proc replaceFileNameTokens { fileName name buildName platformName } { + # + # NOTE: Returns the specified file name containing the platform name instead + # of platform placeholder tokens. + # + return [string map [list <build> $buildName <platform> $platformName \ + <name> $name] $fileName] +} + +proc substFile { fileName } { + # + # NOTE: Performs all Tcl command, variable, and backslash substitutions in + # the specified file and then rewrites the contents of that same file + # with the substituted data. + # + return [writeFile $fileName [uplevel 1 [list subst [readFile $fileName]]]] +} + +# +# NOTE: This is the entry point for this script. +# +set script [file normalize [info script]] + +if {[string length $script] == 0} then { + fail "script file currently being evaluated is unknown" true +} + +set path [file dirname $script] +set rootName [file rootname [file tail $script]] + +############################################################################### + +# +# NOTE: Process and verify all the command line arguments. +# +set argc [llength $argv] +if {$argc < 1 || $argc > 5} then {fail} + +set binaryDirectory [lindex $argv 0] + +if {[string length $binaryDirectory] == 0} then { + fail "invalid binary directory" +} + +if {![file exists $binaryDirectory] || \ + ![file isdirectory $binaryDirectory]} then { + fail "binary directory does not exist" +} + +if {$argc >= 2} then { + set sourceDirectory [lindex $argv 1] +} else { + # + # NOTE: Assume that the source directory is the parent directory of the one + # that contains this script file. + # + set sourceDirectory [file dirname $path] +} + +if {[string length $sourceDirectory] == 0} then { + fail "invalid source directory" +} + +if {![file exists $sourceDirectory] || \ + ![file isdirectory $sourceDirectory]} then { + fail "source directory does not exist" +} + +if {$argc >= 3} then { + set packageFlavor [lindex $argv 2] +} else { + # + # NOTE: Assume the package flavor is WinRT. + # + set packageFlavor WinRT +} + +if {[string length $packageFlavor] == 0} then { + fail "invalid package flavor" +} + +if {$argc >= 4} then { + set platformNames [list] + + foreach platformName [split [lindex $argv 3] ", "] { + set platformName [string trim $platformName] + + if {[string length $platformName] > 0} then { + lappend platformNames $platformName + } + } +} + +if {$argc >= 5} then { + set vsVersion [lindex $argv 4] +} else { + set vsVersion 2012 +} + +if {[string length $vsVersion] == 0} then { + fail "invalid Visual Studio version" +} + +if {![string equal $vsVersion 2012] && ![string equal $vsVersion 2013] && \ + ![string equal $vsVersion 2015]} then { + fail [appendArgs \ + "unsupported Visual Studio version, must be one of: " \ + [list 2012 2013 2015]] +} + +set shortNames(WinRT,2012) SQLite.WinRT +set shortNames(WinRT,2013) SQLite.WinRT.2013 +set shortNames(WinRT81,2013) SQLite.WinRT81 +set shortNames(WP80,2012) SQLite.WP80 +set shortNames(WP80,2013) SQLite.WP80.2013 +set shortNames(WP81,2013) SQLite.WP81 +set shortNames(Win32,2012) SQLite.Win32 +set shortNames(Win32,2013) SQLite.Win32.2013 +set shortNames(UWP,2015) SQLite.UWP.2015 + +set displayNames(WinRT,2012) "SQLite for Windows Runtime" +set displayNames(WinRT,2013) "SQLite for Windows Runtime" +set displayNames(WinRT81,2013) "SQLite for Windows Runtime (Windows 8.1)" +set displayNames(WP80,2012) "SQLite for Windows Phone" +set displayNames(WP80,2013) "SQLite for Windows Phone" +set displayNames(WP81,2013) "SQLite for Windows Phone 8.1" +set displayNames(Win32,2012) "SQLite for Windows" +set displayNames(Win32,2013) "SQLite for Windows" +set displayNames(UWP,2015) "SQLite for Universal Windows Platform" + +if {[string equal $packageFlavor WinRT]} then { + set shortName $shortNames($packageFlavor,$vsVersion) + set displayName $displayNames($packageFlavor,$vsVersion) + set targetPlatformIdentifier Windows + set targetPlatformVersion v8.0 + set minVsVersion [getMinVsVersionXmlChunk $vsVersion] + set maxPlatformVersion \ + [getMaxPlatformVersionXmlChunk $packageFlavor $vsVersion] + set extraSdkPath "" + set extraFileListAttributes \ + [getExtraFileListXmlChunk $packageFlavor $vsVersion] +} elseif {[string equal $packageFlavor WinRT81]} then { + if {$vsVersion ne "2013"} then { + fail [appendArgs \ + "unsupported combination, package flavor " $packageFlavor \ + " is only supported with Visual Studio 2013"] + } + set shortName $shortNames($packageFlavor,$vsVersion) + set displayName $displayNames($packageFlavor,$vsVersion) + set targetPlatformIdentifier Windows + set targetPlatformVersion v8.1 + set minVsVersion [getMinVsVersionXmlChunk $vsVersion] + set maxPlatformVersion \ + [getMaxPlatformVersionXmlChunk $packageFlavor $vsVersion] + set extraSdkPath "" + set extraFileListAttributes \ + [getExtraFileListXmlChunk $packageFlavor $vsVersion] +} elseif {[string equal $packageFlavor WP80]} then { + set shortName $shortNames($packageFlavor,$vsVersion) + set displayName $displayNames($packageFlavor,$vsVersion) + set targetPlatformIdentifier "Windows Phone" + set targetPlatformVersion v8.0 + set minVsVersion [getMinVsVersionXmlChunk $vsVersion] + set maxPlatformVersion \ + [getMaxPlatformVersionXmlChunk $packageFlavor $vsVersion] + set extraSdkPath "\\..\\$targetPlatformIdentifier" + set extraFileListAttributes \ + [getExtraFileListXmlChunk $packageFlavor $vsVersion] +} elseif {[string equal $packageFlavor WP81]} then { + if {$vsVersion ne "2013"} then { + fail [appendArgs \ + "unsupported combination, package flavor " $packageFlavor \ + " is only supported with Visual Studio 2013"] + } + set shortName $shortNames($packageFlavor,$vsVersion) + set displayName $displayNames($packageFlavor,$vsVersion) + set targetPlatformIdentifier WindowsPhoneApp + set targetPlatformVersion v8.1 + set minVsVersion [getMinVsVersionXmlChunk $vsVersion] + set maxPlatformVersion \ + [getMaxPlatformVersionXmlChunk $packageFlavor $vsVersion] + set extraSdkPath "\\..\\$targetPlatformIdentifier" + set extraFileListAttributes \ + [getExtraFileListXmlChunk $packageFlavor $vsVersion] +} elseif {[string equal $packageFlavor UWP]} then { + if {$vsVersion ne "2015"} then { + fail [appendArgs \ + "unsupported combination, package flavor " $packageFlavor \ + " is only supported with Visual Studio 2015"] + } + set shortName $shortNames($packageFlavor,$vsVersion) + set displayName $displayNames($packageFlavor,$vsVersion) + set targetPlatformIdentifier UAP; # NOTE: Not "UWP". + set targetPlatformVersion v0.8.0.0 + set minVsVersion [getMinVsVersionXmlChunk $vsVersion] + set maxPlatformVersion \ + [getMaxPlatformVersionXmlChunk $packageFlavor $vsVersion] + set extraSdkPath "\\..\\$targetPlatformIdentifier" + set extraFileListAttributes \ + [getExtraFileListXmlChunk $packageFlavor $vsVersion] +} elseif {[string equal $packageFlavor Win32]} then { + set shortName $shortNames($packageFlavor,$vsVersion) + set displayName $displayNames($packageFlavor,$vsVersion) + set targetPlatformIdentifier Windows + set targetPlatformVersion v8.0 + set minVsVersion [getMinVsVersionXmlChunk $vsVersion] + set maxPlatformVersion \ + [getMaxPlatformVersionXmlChunk $packageFlavor $vsVersion] + set extraSdkPath "" + set extraFileListAttributes \ + [getExtraFileListXmlChunk $packageFlavor $vsVersion] +} else { + fail [appendArgs \ + "unsupported package flavor, must be one of: " \ + [list WinRT WinRT81 WP80 WP81 UWP Win32]] +} + +############################################################################### + +# +# NOTE: Evaluate the user-specific customizations file, if it exists. +# +set userFile [file join $path [appendArgs \ + $rootName . $tcl_platform(user) .tcl]] + +if {[file exists $userFile] && \ + [file isfile $userFile]} then { + source $userFile +} + +############################################################################### + +set templateFile [file join $path win sqlite.vsix] + +if {![file exists $templateFile] || \ + ![file isfile $templateFile]} then { + fail [appendArgs "template file \"" $templateFile "\" does not exist"] +} + +set currentDirectory [pwd] +set outputFile [file join $currentDirectory [appendArgs sqlite- \ + $packageFlavor -output.vsix]] + +if {[file exists $outputFile]} then { + fail [appendArgs "output file \"" $outputFile "\" already exists"] +} + +############################################################################### + +# +# NOTE: Make sure that a valid temporary directory exists. +# +set temporaryDirectory [getTemporaryPath] + +if {[string length $temporaryDirectory] == 0 || \ + ![file exists $temporaryDirectory] || \ + ![file isdirectory $temporaryDirectory]} then { + fail "cannot locate a usable temporary directory" +} + +# +# NOTE: Setup the staging directory to have a unique name inside of the +# configured temporary directory. +# +set stagingDirectory [file normalize [file join $temporaryDirectory \ + [appendArgs $rootName . [pid]]]] + +############################################################################### + +# +# NOTE: Configure the external zipping tool. First, see if it has already +# been pre-configured. If not, try to query it from the environment. +# Finally, fallback on the default of simply "zip", which will then +# be assumed to exist somewhere along the PATH. +# +if {![info exists zip]} then { + if {[info exists env(ZipTool)]} then { + set zip $env(ZipTool) + } + if {![info exists zip] || ![file exists $zip]} then { + set zip zip + } +} + +# +# NOTE: Configure the external unzipping tool. First, see if it has already +# been pre-configured. If not, try to query it from the environment. +# Finally, fallback on the default of simply "unzip", which will then +# be assumed to exist somewhere along the PATH. +# +if {![info exists unzip]} then { + if {[info exists env(UnZipTool)]} then { + set unzip $env(UnZipTool) + } + if {![info exists unzip] || ![file exists $unzip]} then { + set unzip unzip + } +} + +############################################################################### + +# +# NOTE: Attempt to extract the SQLite version from the "sqlite3.h" header file +# in the source directory. This script assumes that the header file has +# already been generated by the build process. +# +set pattern {^#define\s+SQLITE_VERSION\s+"(.*)"$} +set data [readFile [file join $sourceDirectory sqlite3.h]] + +if {![regexp -line -- $pattern $data dummy version]} then { + fail [appendArgs "cannot locate SQLITE_VERSION value in \"" \ + [file join $sourceDirectory sqlite3.h] \"] +} + +############################################################################### + +# +# NOTE: Setup all the master file list data. This includes the source file +# names, the destination file names, and the file processing flags. The +# possible file processing flags are: +# +# "buildNeutral" -- This flag indicates the file location and content do +# not depend on the build configuration. +# +# "platformNeutral" -- This flag indicates the file location and content +# do not depend on the build platform. +# +# "subst" -- This flag indicates that the file contains dynamic textual +# content that needs to be processed using [subst] prior to +# packaging the file into the final VSIX package. The primary +# use of this flag is to insert the name of the VSIX package, +# some package flavor-specific value, or the SQLite version +# into a file. +# +# "noDebug" -- This flag indicates that the file should be skipped when +# processing the debug build. +# +# "noRetail" -- This flag indicates that the file should be skipped when +# processing the retail build. +# +# "move" -- This flag indicates that the file should be moved from the +# source to the destination instead of being copied. +# +# This file metadata may be overridden, either in whole or in part, via +# the user-specific customizations file. +# +if {![info exists fileNames(source)]} then { + set fileNames(source) [list "" "" \ + [file join $stagingDirectory DesignTime <build> <platform> sqlite3.props] \ + [file join $sourceDirectory sqlite3.h] \ + [file join $binaryDirectory <build> <platform> sqlite3.lib] \ + [file join $binaryDirectory <build> <platform> sqlite3.dll]] + + if {![info exists no(symbols)]} then { + lappend fileNames(source) \ + [file join $binaryDirectory <build> <platform> sqlite3.pdb] + } +} + +if {![info exists fileNames(destination)]} then { + set fileNames(destination) [list \ + [file join $stagingDirectory extension.vsixmanifest] \ + [file join $stagingDirectory SDKManifest.xml] \ + [file join $stagingDirectory DesignTime <build> <platform> <name>.props] \ + [file join $stagingDirectory DesignTime <build> <platform> sqlite3.h] \ + [file join $stagingDirectory DesignTime <build> <platform> sqlite3.lib] \ + [file join $stagingDirectory Redist <build> <platform> sqlite3.dll]] + + if {![info exists no(symbols)]} then { + lappend fileNames(destination) \ + [file join $stagingDirectory Redist <build> <platform> sqlite3.pdb] + } +} + +if {![info exists fileNames(flags)]} then { + set fileNames(flags) [list \ + [list buildNeutral platformNeutral subst] \ + [list buildNeutral platformNeutral subst] \ + [list buildNeutral platformNeutral subst move] \ + [list buildNeutral platformNeutral] \ + [list] [list] [list noRetail]] + + if {![info exists no(symbols)]} then { + lappend fileNames(flags) [list noRetail] + } +} + +############################################################################### + +# +# NOTE: Setup the list of builds supported by this script. These may be +# overridden via the user-specific customizations file. +# +if {![info exists buildNames]} then { + set buildNames [list Debug Retail] +} + +############################################################################### + +# +# NOTE: Setup the list of platforms supported by this script. These may be +# overridden via the command line or the user-specific customizations +# file. +# +if {![info exists platformNames] || [llength $platformNames] == 0} then { + set platformNames [list x86 x64 ARM] +} + +############################################################################### + +# +# NOTE: Make sure the staging directory exists, creating it if necessary. +# +file mkdir $stagingDirectory + +# +# NOTE: Build the Tcl command used to extract the template VSIX package to +# the staging directory. +# +set extractCommand [list exec -- $unzip $templateFile -d $stagingDirectory] + +# +# NOTE: Extract the template VSIX package to the staging directory. +# +eval $extractCommand + +############################################################################### + +# +# NOTE: Process each file in the master file list. There are actually three +# parallel lists that contain the source file names, the destination file +# names, and the file processing flags. If the "buildNeutral" flag is +# present, the file location and content do not depend on the build +# configuration and "CommonConfiguration" will be used in place of the +# build configuration name. If the "platformNeutral" flag is present, +# the file location and content do not depend on the build platform and +# "neutral" will be used in place of the build platform name. If the +# "subst" flag is present, the file is assumed to be a text file that may +# contain Tcl variable, command, and backslash replacements, to be +# dynamically replaced during processing using the Tcl [subst] command. +# If the "noDebug" flag is present, the file will be skipped when +# processing for the debug build. If the "noRetail" flag is present, the +# file will be skipped when processing for the retail build. If the +# "move" flag is present, the source file will be deleted after it is +# copied to the destination file. If the source file name is an empty +# string, the destination file name will be assumed to already exist in +# the staging directory and will not be copied; however, Tcl variable, +# command, and backslash replacements may still be performed on the +# destination file prior to the final VSIX package being built if the +# "subst" flag is present. +# +foreach sourceFileName $fileNames(source) \ + destinationFileName $fileNames(destination) \ + fileFlags $fileNames(flags) { + # + # NOTE: Process the file flags into separate boolean variables that may be + # used within the loop. + # + set isBuildNeutral [expr {[lsearch $fileFlags buildNeutral] != -1}] + set isPlatformNeutral [expr {[lsearch $fileFlags platformNeutral] != -1}] + set isMove [expr {[lsearch $fileFlags move] != -1}] + set useSubst [expr {[lsearch $fileFlags subst] != -1}] + + # + # NOTE: If the current file is build-neutral, then only one build will + # be processed for it, namely "CommonConfiguration"; otherwise, each + # supported build will be processed for it individually. + # + foreach buildName \ + [expr {$isBuildNeutral ? [list CommonConfiguration] : $buildNames}] { + # + # NOTE: Should the current file be skipped for this build? + # + if {[lsearch $fileFlags no${buildName}] != -1} then { + continue + } + + # + # NOTE: If the current file is platform-neutral, then only one platform + # will be processed for it, namely "neutral"; otherwise, each + # supported platform will be processed for it individually. + # + foreach platformName \ + [expr {$isPlatformNeutral ? [list neutral] : $platformNames}] { + # + # NOTE: Use the actual platform name in the destination file name. + # + set newDestinationFileName [replaceFileNameTokens $destinationFileName \ + $shortName $buildName $platformName] + + # + # NOTE: Does the source file need to be copied to the destination file? + # + if {[string length $sourceFileName] > 0} then { + # + # NOTE: First, make sure the destination directory exists. + # + file mkdir [file dirname $newDestinationFileName] + + # + # NOTE: Then, copy the source file to the destination file verbatim. + # + set newSourceFileName [replaceFileNameTokens $sourceFileName \ + $shortName $buildName $platformName] + + file copy $newSourceFileName $newDestinationFileName + + # + # NOTE: If this is a move instead of a copy, delete the source file + # now. + # + if {$isMove} then { + file delete $newSourceFileName + } + } + + # + # NOTE: Does the destination file contain dynamic replacements that must + # be processed now? + # + if {$useSubst} then { + # + # NOTE: Perform any dynamic replacements contained in the destination + # file and then re-write it in-place. + # + substFile $newDestinationFileName + } + } + } +} + +############################################################################### + +# +# NOTE: Change the current directory to the staging directory so that the +# external archive building tool can pickup the necessary files using +# relative paths. +# +cd $stagingDirectory + +# +# NOTE: Build the Tcl command used to archive the final VSIX package in the +# output directory. +# +set archiveCommand [list exec -- $zip -r $outputFile *] + +# +# NOTE: Build the final VSIX package archive in the output directory. +# +eval $archiveCommand + +# +# NOTE: Change back to the previously saved current directory. +# +cd $currentDirectory + +# +# NOTE: Cleanup the temporary staging directory. +# +file delete -force $stagingDirectory + +############################################################################### + +# +# NOTE: Success, emit the fully qualified path of the generated VSIX file. +# +puts stdout $outputFile diff --git a/tool/offsets.c b/tool/offsets.c new file mode 100644 index 0000000..26ee9fc --- /dev/null +++ b/tool/offsets.c @@ -0,0 +1,329 @@ +/* +** This program searches an SQLite database file for the lengths and +** offsets for all TEXT or BLOB entries for a particular column of a +** particular table. The rowid, size and offset for the column are +** written to standard output. There are three arguments, which are the +** name of the database file, the table, and the column. +*/ +#include "sqlite3.h" +#include <stdio.h> +#include <stdlib.h> +#include <stdarg.h> +#include <string.h> + +typedef unsigned char u8; +typedef struct GState GState; + +#define ArraySize(X) (sizeof(X)/sizeof(X[0])) + +/* +** Global state information for this program. +*/ +struct GState { + char *zErr; /* Error message text */ + FILE *f; /* Open database file */ + int szPg; /* Page size for the database file */ + int iRoot; /* Root page of the table */ + int iCol; /* Column number for the column */ + int pgno; /* Current page number */ + u8 *aPage; /* Current page content */ + u8 *aStack[20]; /* Page stack */ + int aPgno[20]; /* Page number stack */ + int nStack; /* Depth of stack */ + int bTrace; /* True for tracing output */ +}; + +/* +** Write an error. +*/ +static void ofstError(GState *p, const char *zFormat, ...){ + va_list ap; + sqlite3_free(p->zErr); + va_start(ap, zFormat); + p->zErr = sqlite3_vmprintf(zFormat, ap); + va_end(ap); +} + +/* +** Write a trace message +*/ +static void ofstTrace(GState *p, const char *zFormat, ...){ + va_list ap; + if( p->bTrace ){ + va_start(ap, zFormat); + vprintf(zFormat, ap); + va_end(ap); + } +} + +/* +** Find the root page of the table and the column number of the column. +*/ +static void ofstRootAndColumn( + GState *p, /* Global state */ + const char *zFile, /* Name of the database file */ + const char *zTable, /* Name of the table */ + const char *zColumn /* Name of the column */ +){ + sqlite3 *db = 0; + sqlite3_stmt *pStmt = 0; + char *zSql = 0; + int rc; + if( p->zErr ) return; + rc = sqlite3_open(zFile, &db); + if( rc ){ + ofstError(p, "cannot open database file \"%s\"", zFile); + goto rootAndColumn_exit; + } + zSql = sqlite3_mprintf("SELECT rootpage FROM sqlite_schema WHERE name=%Q", + zTable); + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc ) ofstError(p, "%s: [%s]", sqlite3_errmsg(db), zSql); + sqlite3_free(zSql); + if( p->zErr ) goto rootAndColumn_exit; + if( sqlite3_step(pStmt)!=SQLITE_ROW ){ + ofstError(p, "cannot find table [%s]\n", zTable); + sqlite3_finalize(pStmt); + goto rootAndColumn_exit; + } + p->iRoot = sqlite3_column_int(pStmt , 0); + sqlite3_finalize(pStmt); + + p->iCol = -1; + zSql = sqlite3_mprintf("PRAGMA table_info(%Q)", zTable); + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc ) ofstError(p, "%s: [%s}", sqlite3_errmsg(db), zSql); + sqlite3_free(zSql); + if( p->zErr ) goto rootAndColumn_exit; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + const char *zCol = sqlite3_column_text(pStmt, 1); + if( strlen(zCol)==strlen(zColumn) + && sqlite3_strnicmp(zCol, zColumn, strlen(zCol))==0 + ){ + p->iCol = sqlite3_column_int(pStmt, 0); + break; + } + } + sqlite3_finalize(pStmt); + if( p->iCol<0 ){ + ofstError(p, "no such column: %s.%s", zTable, zColumn); + goto rootAndColumn_exit; + } + + zSql = sqlite3_mprintf("PRAGMA page_size"); + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc ) ofstError(p, "%s: [%s]", sqlite3_errmsg(db), zSql); + sqlite3_free(zSql); + if( p->zErr ) goto rootAndColumn_exit; + if( sqlite3_step(pStmt)!=SQLITE_ROW ){ + ofstError(p, "cannot find page size"); + }else{ + p->szPg = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + +rootAndColumn_exit: + sqlite3_close(db); + return; +} + +/* +** Pop a page from the stack +*/ +static void ofstPopPage(GState *p){ + if( p->nStack<=0 ) return; + p->nStack--; + sqlite3_free(p->aStack[p->nStack]); + p->pgno = p->aPgno[p->nStack-1]; + p->aPage = p->aStack[p->nStack-1]; +} + + +/* +** Push a new page onto the stack. +*/ +static void ofstPushPage(GState *p, int pgno){ + u8 *pPage; + size_t got; + if( p->zErr ) return; + if( p->nStack >= ArraySize(p->aStack) ){ + ofstError(p, "page stack overflow"); + return; + } + p->aPgno[p->nStack] = pgno; + p->aStack[p->nStack] = pPage = sqlite3_malloc( p->szPg ); + if( pPage==0 ){ + fprintf(stderr, "out of memory\n"); + exit(1); + } + p->nStack++; + p->aPage = pPage; + p->pgno = pgno; + fseek(p->f, (pgno-1)*p->szPg, SEEK_SET); + got = fread(pPage, 1, p->szPg, p->f); + if( got!=p->szPg ){ + ofstError(p, "unable to read page %d", pgno); + ofstPopPage(p); + } +} + +/* Read a two-byte integer at the given offset into the current page */ +static int ofst2byte(GState *p, int ofst){ + int x = p->aPage[ofst]; + return (x<<8) + p->aPage[ofst+1]; +} + +/* Read a four-byte integer at the given offset into the current page */ +static int ofst4byte(GState *p, int ofst){ + int x = p->aPage[ofst]; + x = (x<<8) + p->aPage[ofst+1]; + x = (x<<8) + p->aPage[ofst+2]; + x = (x<<8) + p->aPage[ofst+3]; + return x; +} + +/* Read a variable-length integer. Update the offset */ +static sqlite3_int64 ofstVarint(GState *p, int *pOfst){ + sqlite3_int64 x = 0; + u8 *a = &p->aPage[*pOfst]; + int n = 0; + while( n<8 && (a[0] & 0x80)!=0 ){ + x = (x<<7) + (a[0] & 0x7f); + n++; + a++; + } + if( n==8 ){ + x = (x<<8) + a[0]; + }else{ + x = (x<<7) + a[0]; + } + *pOfst += (n+1); + return x; +} + +/* Return the absolute offset into a file for the given offset +** into the current page */ +static int ofstInFile(GState *p, int ofst){ + return p->szPg*(p->pgno-1) + ofst; +} + +/* Return the size (in bytes) of the data corresponding to the +** given serial code */ +static int ofstSerialSize(int scode){ + if( scode<5 ) return scode; + if( scode==5 ) return 6; + if( scode<8 ) return 8; + if( scode<12 ) return 0; + return (scode-12)/2; +} + +/* Forward reference */ +static void ofstWalkPage(GState*, int); + +/* Walk an interior btree page */ +static void ofstWalkInteriorPage(GState *p){ + int nCell; + int i; + int ofst; + int iChild; + + nCell = ofst2byte(p, 3); + for(i=0; i<nCell; i++){ + ofst = ofst2byte(p, 12+i*2); + iChild = ofst4byte(p, ofst); + ofstWalkPage(p, iChild); + if( p->zErr ) return; + } + ofstWalkPage(p, ofst4byte(p, 8)); +} + +/* Walk a leaf btree page */ +static void ofstWalkLeafPage(GState *p){ + int nCell; + int i; + int ofst; + int nPayload; + sqlite3_int64 rowid; + int nHdr; + int j; + int scode; + int sz; + int dataOfst; + char zMsg[200]; + + nCell = ofst2byte(p, 3); + for(i=0; i<nCell; i++){ + ofst = ofst2byte(p, 8+i*2); + nPayload = ofstVarint(p, &ofst); + rowid = ofstVarint(p, &ofst); + if( nPayload > p->szPg-35 ){ + sqlite3_snprintf(sizeof(zMsg), zMsg, + "# overflow rowid %lld", rowid); + printf("%s\n", zMsg); + continue; + } + dataOfst = ofst; + nHdr = ofstVarint(p, &ofst); + dataOfst += nHdr; + for(j=0; j<p->iCol; j++){ + scode = ofstVarint(p, &ofst); + dataOfst += ofstSerialSize(scode); + } + scode = ofstVarint(p, &ofst); + sz = ofstSerialSize(scode); + sqlite3_snprintf(sizeof(zMsg), zMsg, + "rowid %12lld size %5d offset %8d", + rowid, sz, ofstInFile(p, dataOfst)); + printf("%s\n", zMsg); + } +} + +/* +** Output results from a single page. +*/ +static void ofstWalkPage(GState *p, int pgno){ + if( p->zErr ) return; + ofstPushPage(p, pgno); + if( p->zErr ) return; + if( p->aPage[0]==5 ){ + ofstWalkInteriorPage(p); + }else if( p->aPage[0]==13 ){ + ofstWalkLeafPage(p); + }else{ + ofstError(p, "page %d has a faulty type byte: %d", pgno, p->aPage[0]); + } + ofstPopPage(p); +} + +int main(int argc, char **argv){ + GState g; + memset(&g, 0, sizeof(g)); + if( argc>2 && strcmp(argv[1],"--trace")==0 ){ + g.bTrace = 1; + argc--; + argv++; + } + if( argc!=4 ){ + fprintf(stderr, "Usage: %s DATABASE TABLE COLUMN\n", *argv); + exit(1); + } + ofstRootAndColumn(&g, argv[1], argv[2], argv[3]); + if( g.zErr ){ + fprintf(stderr, "%s\n", g.zErr); + exit(1); + } + ofstTrace(&g, "# szPg = %d\n", g.szPg); + ofstTrace(&g, "# iRoot = %d\n", g.iRoot); + ofstTrace(&g, "# iCol = %d\n", g.iCol); + g.f = fopen(argv[1], "rb"); + if( g.f==0 ){ + fprintf(stderr, "cannot open \"%s\"\n", argv[1]); + exit(1); + } + ofstWalkPage(&g, g.iRoot); + if( g.zErr ){ + fprintf(stderr, "%s\n", g.zErr); + exit(1); + } + return 0; +} diff --git a/tool/omittest-msvc.tcl b/tool/omittest-msvc.tcl new file mode 100644 index 0000000..28c4b98 --- /dev/null +++ b/tool/omittest-msvc.tcl @@ -0,0 +1,99 @@ +# Run this TCL script in order to build using MSVC multiple times +# with various compile-time options. Use this to verify that the various +# compile-time options all work with MSVC. +# +set OPTIONS [list \ + SQLITE_ALLOW_ROWID_IN_VIEW \ + SQLITE_ENABLE_COLUMN_METADATA \ + SQLITE_ENABLE_EXPENSIVE_ASSERT \ + SQLITE_ENABLE_IOTRACE \ + SQLITE_ENABLE_MEMORY_MANAGEMENT \ + SQLITE_ENABLE_MEMSYS3 \ + SQLITE_ENABLE_MEMSYS5 \ + SQLITE_ENABLE_OVERSIZE_CELL_CHECK \ + SQLITE_ENABLE_UNLOCK_NOTIFY \ + SQLITE_ENABLE_UPDATE_DELETE_LIMIT \ + SQLITE_OMIT_ALTERTABLE-x \ + SQLITE_OMIT_ATTACH-x \ + SQLITE_OMIT_AUTHORIZATION \ + SQLITE_OMIT_AUTOINCREMENT \ + SQLITE_OMIT_AUTOINIT \ + SQLITE_OMIT_AUTOMATIC_INDEX \ + SQLITE_OMIT_AUTORESET \ + SQLITE_OMIT_AUTOVACUUM \ + SQLITE_OMIT_BETWEEN_OPTIMIZATION \ + SQLITE_OMIT_BLOB_LITERAL \ + SQLITE_OMIT_BTREECOUNT \ + SQLITE_OMIT_CASE_SENSITIVE_LIKE_PRAGMA \ + SQLITE_OMIT_CAST \ + SQLITE_OMIT_CHECK \ + SQLITE_OMIT_COMPILEOPTION_DIAGS \ + SQLITE_OMIT_COMPLETE \ + SQLITE_OMIT_COMPOUND_SELECT \ + SQLITE_OMIT_CONFLICT_CLAUSE \ + SQLITE_OMIT_CTE \ + SQLITE_OMIT_DATETIME_FUNCS \ + SQLITE_OMIT_DECLTYPE \ + SQLITE_OMIT_DEPRECATED \ + SQLITE_OMIT_DESERIALIZE \ + SQLITE_OMIT_DISKIO-x \ + SQLITE_OMIT_EXPLAIN-x \ + SQLITE_OMIT_FLAG_PRAGMAS \ + SQLITE_OMIT_FLOATING_POINT \ + SQLITE_OMIT_FOREIGN_KEY \ + SQLITE_OMIT_GENERATED_COLUMNS \ + SQLITE_OMIT_GET_TABLE \ + SQLITE_OMIT_HEX_INTEGER \ + SQLITE_OMIT_INCRBLOB-x \ + SQLITE_OMIT_INTEGRITY_CHECK \ + SQLITE_OMIT_INTROSPECTION_PRAGMAS \ + SQLITE_OMIT_JSON \ + SQLITE_OMIT_LIKE_OPTIMIZATION \ + SQLITE_OMIT_LOAD_EXTENSION \ + SQLITE_OMIT_LOCALTIME \ + SQLITE_OMIT_LOOKASIDE \ + SQLITE_OMIT_MEMORYDB \ + SQLITE_OMIT_OR_OPTIMIZATION \ + SQLITE_OMIT_PAGER_PRAGMAS-x \ + SQLITE_OMIT_PARSER_TRACE \ + SQLITE_OMIT_POPEN \ + SQLITE_OMIT_PRAGMA-x \ + SQLITE_OMIT_PROGRESS_CALLBACK \ + SQLITE_OMIT_QUICKBALANCE \ + SQLITE_OMIT_RANDOMNESS \ + SQLITE_OMIT_REINDEX-x \ + SQLITE_OMIT_SCHEMA_PRAGMAS \ + SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS \ + SQLITE_OMIT_SHARED_CACHE \ + SQLITE_OMIT_SHUTDOWN_DIRECTORIES \ + SQLITE_OMIT_SUBQUERY-x \ + SQLITE_OMIT_TCL_VARIABLE \ + SQLITE_OMIT_TEMPDB \ + SQLITE_OMIT_TEST_CONTROL \ + SQLITE_OMIT_TRACE \ + SQLITE_OMIT_TRIGGER \ + SQLITE_OMIT_TRUNCATE_OPTIMIZATION \ + SQLITE_OMIT_UPSERT \ + SQLITE_OMIT_UTF16 \ + SQLITE_OMIT_VACUUM-x \ + SQLITE_OMIT_VIEW-x \ + SQLITE_OMIT_VIRTUALTABLE-x \ + SQLITE_OMIT_WAL \ + SQLITE_OMIT_WINDOWFUNC-x \ + SQLITE_OMIT_WSD \ + SQLITE_OMIT_XFER_OPT \ +] +set start [lindex $argv 0] +foreach opt $OPTIONS { + if {[regexp x $opt]} continue + if {[string compare $opt $start]<0} continue + exec fossil clean -x + set cmd "nmake /f Makefile.msc USE_FULLWARN=0 OPTIMIZATIONS=0" + append cmd " CCOPTS=-D$opt" + puts $cmd + set res [catch {eval exec $cmd} outtxt] + if {[regexp {sqlite3.c.\d+.: error} $outtxt]} { + puts "FAILED:\n$outtxt" + exit + } +} diff --git a/tool/omittest.tcl b/tool/omittest.tcl new file mode 100644 index 0000000..8862c68 --- /dev/null +++ b/tool/omittest.tcl @@ -0,0 +1,370 @@ +# Documentation for this script. This may be output to stderr +# if the script is invoked incorrectly. +set ::USAGE_MESSAGE { +This Tcl script is used to test the various compile time options +available for omitting code (the SQLITE_OMIT_xxx options). It +should be invoked as follows: + + <script> ?test-symbol? ?-makefile PATH-TO-MAKEFILE? ?-skip_run? + +The default value for ::MAKEFILE is "../Makefile.linux.gcc". + +If -skip_run option is given then only the compile part is attempted. + +This script builds the testfixture program and runs the SQLite test suite +once with each SQLITE_OMIT_ option defined and then once with all options +defined together. Each run is performed in a seperate directory created +as a sub-directory of the current directory by the script. The output +of the build is saved in <sub-directory>/build.log. The output of the +test-suite is saved in <sub-directory>/test.log. + +Almost any SQLite makefile (except those generated by configure - see below) +should work. The following properties are required: + + * The makefile should support the "testfixture" target. + * The makefile should support the "test" target. + * The makefile should support the variable "OPTS" as a way to pass + options from the make command line to lemon and the C compiler. + +More precisely, the following two invocations must be supported: + + $::MAKEBIN -f $::MAKEFILE testfixture OPTS="-DSQLITE_OMIT_ALTERTABLE=1" + $::MAKEBIN -f $::MAKEFILE test + +Makefiles generated by the sqlite configure program cannot be used as +they do not respect the OPTS variable. +} + + +# Build a testfixture executable and run quick.test using it. The first +# parameter is the name of the directory to create and use to run the +# test in. The second parameter is a list of OMIT symbols to define +# when doing so. For example: +# +# run_quick_test /tmp/testdir {SQLITE_OMIT_TRIGGER SQLITE_OMIT_VIEW} +# +# +proc run_quick_test {dir omit_symbol_list} { + # Compile the value of the OPTS Makefile variable. + set opts "" + if {$::tcl_platform(platform)=="windows"} { + append opts "OPTS += -DSQLITE_OS_WIN=1\n" + set target "testfixture.exe" + } else { + append opts "OPTS += -DSQLITE_OS_UNIX=1\n" + } + foreach sym $omit_symbol_list { + append opts "OPTS += -D${sym}=1\n" + } + + # Create the directory and do the build. If an error occurs return + # early without attempting to run the test suite. + file mkdir $dir + puts -nonewline "Building $dir..." + flush stdout + catch { + file copy -force ./config.h $dir + file copy -force ./libtool $dir + } + set fd [open $::MAKEFILE] + set mkfile [read $fd] + close $fd + regsub {\ninclude} $mkfile "\n$opts\ninclude" mkfile + set fd [open $dir/makefile w] + puts $fd $mkfile + close $fd + + set rc [catch { + exec $::MAKEBIN -C $dir -f makefile clean $::TARGET >& $dir/build.log + }] + if {$rc} { + puts "No good. See $dir/build.log." + return + } else { + puts "Ok" + } + + # Create an empty file "$dir/sqlite3". This is to trick the makefile out + # of trying to build the sqlite shell. The sqlite shell won't build + # with some of the OMIT options (i.e OMIT_COMPLETE). + set sqlite3_dummy $dir/sqlite3 + if {$::tcl_platform(platform)=="windows"} { + append sqlite3_dummy ".exe" + } + if {![file exists $sqlite3_dummy]} { + set wr [open $sqlite3_dummy w] + puts $wr "dummy" + close $wr + } + + if {$::SKIP_RUN} { + # puts "Skip testing $dir." + } else { + # Run the test suite. + puts -nonewline "Testing $dir..." + flush stdout + set rc [catch { + exec $::MAKEBIN -C $dir -f makefile test >& $dir/test.log + }] + if {$rc} { + puts "No good. See $dir/test.log." + } else { + puts "Ok" + } + } +} + + +# This proc processes the command line options passed to this script. +# Currently the only option supported is "-makefile", default +# "../Makefile.linux-gcc". Set the ::MAKEFILE variable to the value of this +# option. +# +proc process_options {argv} { + set ::MAKEBIN make ;# Default value + if {$::tcl_platform(platform)=="windows"} { + set ::MAKEFILE ./Makefile ;# Default value on Windows + } else { + set ::MAKEFILE ./Makefile.linux-gcc ;# Default value + } + set ::SKIP_RUN 1 ;# Default to attempt test + set ::TARGET testfixture ;# Default thing to build + + for {set i 0} {$i < [llength $argv]} {incr i} { + switch -regexp -- [lindex $argv $i] { + -{1,2}makefile { + incr i + set ::MAKEFILE [lindex $argv $i] + } + + -{1,2}nmake { + set ::MAKEBIN nmake + set ::MAKEFILE ./Makefile.msc + } + + -{1,2}target { + incr i + set ::TARGET [lindex $argv $i] + } + + -{1,2}skip_run { + set ::SKIP_RUN 1 + } + -{1,2}run { + set ::SKIP_RUN 0 + } + + -{1,2}help { + puts $::USAGE_MESSAGE + exit + } + + -.* { + puts stderr "Unknown option: [lindex $argv i]" + puts stderr $::USAGE_MESSAGE + exit 1 + } + + default { + if {[info exists ::SYMBOL]} { + puts stderr [string trim $::USAGE_MESSAGE] + exit -1 + } + set ::SYMBOL [lindex $argv $i] + } + } + set ::MAKEFILE [file normalize $::MAKEFILE] + } +} + +# Main routine. +# + +proc main {argv} { + # List of SQLITE_OMIT_XXX symbols supported by SQLite. + set ::OMIT_SYMBOLS [list \ + SQLITE_OMIT_ALTERTABLE \ + SQLITE_OMIT_ANALYZE \ + SQLITE_OMIT_ATTACH \ + SQLITE_OMIT_AUTHORIZATION \ + SQLITE_OMIT_AUTOINCREMENT \ + SQLITE_OMIT_AUTOINIT \ + SQLITE_OMIT_AUTOMATIC_INDEX \ + SQLITE_OMIT_AUTORESET \ + SQLITE_OMIT_AUTOVACUUM \ + SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS \ + SQLITE_OMIT_BETWEEN_OPTIMIZATION \ + SQLITE_OMIT_BLOB_LITERAL \ + SQLITE_OMIT_CASE_SENSITIVE_LIKE_PRAGMA \ + SQLITE_OMIT_CAST \ + SQLITE_OMIT_CHECK \ + SQLITE_OMIT_COMPILEOPTION_DIAGS \ + SQLITE_OMIT_COMPLETE \ + SQLITE_OMIT_COMPOUND_SELECT \ + SQLITE_OMIT_CONFLICT_CLAUSE \ + SQLITE_OMIT_CTE \ + SQLITE_OMIT_DATETIME_FUNCS \ + SQLITE_OMIT_DECLTYPE \ + SQLITE_OMIT_DEPRECATED \ + SQLITE_OMIT_DESERIALIZE \ + SQLITE_OMIT_DISKIO \ + SQLITE_OMIT_EXPLAIN \ + SQLITE_OMIT_FLAG_PRAGMAS \ + SQLITE_OMIT_FLOATING_POINT \ + SQLITE_OMIT_FOREIGN_KEY \ + SQLITE_OMIT_GENERATED_COLUMNS \ + SQLITE_OMIT_GET_TABLE \ + SQLITE_OMIT_HEX_INTEGER \ + SQLITE_OMIT_INCRBLOB \ + SQLITE_OMIT_INTEGRITY_CHECK \ + SQLITE_OMIT_INTROSPECTION_PRAGMAS \ + SQLITE_OMIT_JSON \ + SQLITE_OMIT_LIKE_OPTIMIZATION \ + SQLITE_OMIT_LOAD_EXTENSION \ + SQLITE_OMIT_LOCALTIME \ + SQLITE_OMIT_LOOKASIDE \ + SQLITE_OMIT_MEMORYDB \ + SQLITE_OMIT_OR_OPTIMIZATION \ + SQLITE_OMIT_PAGER_PRAGMAS \ + SQLITE_OMIT_PARSER_TRACE \ + SQLITE_OMIT_POPEN \ + SQLITE_OMIT_PRAGMA \ + SQLITE_OMIT_PROGRESS_CALLBACK \ + SQLITE_OMIT_QUICKBALANCE \ + SQLITE_OMIT_RANDOMNESS \ + SQLITE_OMIT_REINDEX \ + SQLITE_OMIT_SCHEMA_PRAGMAS \ + SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS \ + SQLITE_OMIT_SHARED_CACHE \ + SQLITE_OMIT_SHUTDOWN_DIRECTORIES \ + SQLITE_OMIT_SUBQUERY \ + SQLITE_OMIT_TCL_VARIABLE \ + SQLITE_OMIT_TEMPDB \ + SQLITE_OMIT_TEST_CONTROL \ + SQLITE_OMIT_TRACE \ + SQLITE_OMIT_TRIGGER \ + SQLITE_OMIT_TRUNCATE_OPTIMIZATION \ + SQLITE_OMIT_TWOSIZE_LOOKASIDE \ + SQLITE_OMIT_UPSERT \ + SQLITE_OMIT_UTF \ + SQLITE_OMIT_VACUUM \ + SQLITE_OMIT_VIEW \ + SQLITE_OMIT_VIRTUALTABLE \ + SQLITE_OMIT_WAL \ + SQLITE_OMIT_WINDOWFUNC \ + SQLITE_OMIT_WSD \ + SQLITE_OMIT_XFER_OPT \ + ] + + set ::ENABLE_SYMBOLS [list \ + SQLITE_ALLOW_ROWID_IN_VIEW \ + SQLITE_DISABLE_DIRSYNC \ + SQLITE_DISABLE_FTS \ + SQLITE_DISABLE_INTRINSIC \ + SQLITE_DISABLE_LFS \ + SQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS \ + SQLITE_DISABLE_SKIPAHEAD_DISTINCT \ + SQLITE_ENABLE_API_ARMOR \ + SQLITE_ENABLE_ATOMIC_WRITE \ + SQLITE_ENABLE_BATCH_ATOMIC_WRITE \ + SQLITE_ENABLE_BYTECODE_VTAB \ + SQLITE_ENABLE_CEROD \ + SQLITE_ENABLE_COLUMN_METADATA \ + SQLITE_ENABLE_COLUMN_USED_MASK \ + SQLITE_ENABLE_COMMENTS \ + SQLITE_ENABLE_CORRUPT_PGNO \ + SQLITE_ENABLE_COSTMULT \ + SQLITE_ENABLE_CURSOR_HINTS \ + SQLITE_ENABLE_DBPAGE_VTAB \ + SQLITE_ENABLE_DBSTAT_VTAB \ + SQLITE_ENABLE_EXPENSIVE_ASSERT \ + SQLITE_ENABLE_EXPLAIN_COMMENTS \ + SQLITE_ENABLE_FTS \ + SQLITE_ENABLE_GEOPOLY \ + SQLITE_ENABLE_HIDDEN_COLUMNS \ + SQLITE_ENABLE_ICU \ + SQLITE_ENABLE_ICU_COLLATIONS \ + SQLITE_ENABLE_INTERNAL_FUNCTIONS \ + SQLITE_ENABLE_IOTRACE \ + SQLITE_ENABLE_LOAD_EXTENSION \ + SQLITE_ENABLE_LOCKING_STYLE \ + SQLITE_ENABLE_MATH_FUNCTIONS \ + SQLITE_ENABLE_MEMORY_MANAGEMENT \ + SQLITE_ENABLE_MEMSYS \ + SQLITE_ENABLE_MODULE_COMMENTS \ + SQLITE_ENABLE_MULTIPLEX \ + SQLITE_ENABLE_MULTITHREADED_CHECKS \ + SQLITE_ENABLE_NORMALIZE \ + SQLITE_ENABLE_NULL_TRIM \ + SQLITE_ENABLE_OFFSET_SQL_FUNC \ + SQLITE_ENABLE_OVERSIZE_CELL_CHECK \ + SQLITE_ENABLE_PREUPDATE_HOOK \ + SQLITE_ENABLE_QPSG \ + SQLITE_ENABLE_RBU \ + SQLITE_ENABLE_RTREE \ + SQLITE_ENABLE_SELECTTRACE \ + SQLITE_ENABLE_SESSION \ + SQLITE_ENABLE_SETLK_TIMEOUT \ + SQLITE_ENABLE_SNAPSHOT \ + SQLITE_ENABLE_SORTER_MMAP\ + SQLITE_ENABLE_SORTER_REFERENCE \ + SQLITE_ENABLE_SORTER_REFERENCES \ + SQLITE_ENABLE_SQLLOG\ + SQLITE_ENABLE_STAT \ + SQLITE_ENABLE_STMT_SCANSTATUS \ + SQLITE_ENABLE_STMTVTAB \ + SQLITE_ENABLE_TREETRACE \ + SQLITE_ENABLE_UNKNOWN_FUNCTION \ + SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION \ + SQLITE_ENABLE_UNLOCK_NOTIFY \ + SQLITE_ENABLE_UPDATE_DELETE_LIMIT \ + SQLITE_ENABLE_URI_00_ERROR \ + SQLITE_ENABLE_VFSTRACE \ + SQLITE_ENABLE_WHERETRACE \ + SQLITE_ENABLE_ZIPVFS \ + ] + + # Process any command line options. + process_options $argv + + if {[info exists ::SYMBOL] } { + set sym $::SYMBOL + + if {[lsearch $::OMIT_SYMBOLS $sym]<0 && [lsearch $::ENABLE_SYMBOLS $sym]<0} { + puts stderr "No such symbol: $sym" + exit -1 + } + + set dirname "test_[regsub -nocase {^x*SQLITE_} $sym {}]" + run_quick_test $dirname $sym + } else { + # First try a test with all OMIT symbols except SQLITE_OMIT_FLOATING_POINT + # and SQLITE_OMIT_PRAGMA defined. The former doesn't work (causes segfaults) + # and the latter is currently incompatible with the test suite (this should + # be fixed, but it will be a lot of work). + set allsyms [list] + foreach s $::OMIT_SYMBOLS { + if {$s!="SQLITE_OMIT_FLOATING_POINT" && $s!="SQLITE_OMIT_PRAGMA"} { + lappend allsyms $s + } + } + run_quick_test test_OMIT_EVERYTHING $allsyms + + # Now try one quick.test with each of the OMIT symbols defined. Included + # are the OMIT_FLOATING_POINT and OMIT_PRAGMA symbols, even though we + # know they will fail. It's good to be reminded of this from time to time. + foreach sym $::OMIT_SYMBOLS { + set dirname "test_[regsub -nocase {^x*SQLITE_} $sym {}]" + run_quick_test $dirname $sym + } + + # Try the ENABLE/DISABLE symbols one at a time. + # We don't do them all at once since some are conflicting. + foreach sym $::ENABLE_SYMBOLS { + set dirname "test_[regsub -nocase {^x*SQLITE_} $sym {}]" + run_quick_test $dirname $sym + } + } +} + +main $argv diff --git a/tool/opcodesum.tcl b/tool/opcodesum.tcl new file mode 100644 index 0000000..47dff32 --- /dev/null +++ b/tool/opcodesum.tcl @@ -0,0 +1,34 @@ +#!/usr/bin/tclsh +# +# Run this script, redirecting input from cachegrind output, to compute the +# number of CPU cycles used by each VDBE opcode. +# +# The cachegrind output should be configured so that it reports a single +# column of Ir at the left margin. Ex: +# +# cg_annotation --show=Ir --auto=yes cachegrind.out.* | tclsh opcodesum.tcl +# +set currentop x +set ncycle(x) 0 +while {![eof stdin]} { + set line [string map {\173 x \175 x \042 x} [gets stdin]] + if {[regexp { \. case OP_.*:} $line]} { + regexp {OP_(.+):} $line all currentop + set ncycle($currentop) 0 + } elseif {[lindex $line 1]=="default:" + && [regexp {really OP_Noop and OP_Explain} $line]} { + break + } elseif {[lindex $line 0]!="."} { + regsub -all {[^0-9]} [lindex $line 0] {} n + if {$n!=""} {incr ncycle($currentop) $n} + } +} +unset ncycle(x) +set results {} +foreach op [lsort [array names ncycle]] { + if {$ncycle($op)==0} continue + lappend results [list $ncycle($op) $op] +} +foreach entry [lsort -index 0 -int -decr $results] { + puts [format {%-16s %10d} [lindex $entry 1] [lindex $entry 0]] +} diff --git a/tool/pagesig.c b/tool/pagesig.c new file mode 100644 index 0000000..540c9d7 --- /dev/null +++ b/tool/pagesig.c @@ -0,0 +1,92 @@ +/* +** 2013-10-01 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Compute hash signatures for every page of a database file. This utility +** program is useful for analyzing the output logs generated by the +** ext/misc/vfslog.c extension. +*/ +#include <stdio.h> +#include <string.h> +#include <assert.h> +#include <stdlib.h> + +/* +** Compute signature for a block of content. +** +** For blocks of 16 or fewer bytes, the signature is just a hex dump of +** the entire block. +** +** For blocks of more than 16 bytes, the signature is a hex dump of the +** first 8 bytes followed by a 64-bit has of the entire block. +*/ +static void vlogSignature(unsigned char *p, int n, char *zCksum){ + unsigned int s0 = 0, s1 = 0; + unsigned int *pI; + int i; + if( n<=16 ){ + for(i=0; i<n; i++) sprintf(zCksum+i*2, "%02x", p[i]); + }else{ + pI = (unsigned int*)p; + for(i=0; i<n-7; i+=8){ + s0 += pI[0] + s1; + s1 += pI[1] + s0; + pI += 2; + } + for(i=0; i<8; i++) sprintf(zCksum+i*2, "%02x", p[i]); + sprintf(zCksum+i*2, "-%08x%08x", s0, s1); + } +} + +/* +** Open a file. Find its page size. Read each page, and compute and +** display the page signature. +*/ +static void computeSigs(const char *zFilename){ + FILE *in = fopen(zFilename, "rb"); + unsigned pgsz; + size_t got; + unsigned n; + unsigned char aBuf[50]; + unsigned char aPage[65536]; + + if( in==0 ){ + fprintf(stderr, "cannot open \"%s\"\n", zFilename); + return; + } + got = fread(aBuf, 1, sizeof(aBuf), in); + if( got!=sizeof(aBuf) ){ + goto endComputeSigs; + } + pgsz = aBuf[16]*256 + aBuf[17]; + if( pgsz==1 ) pgsz = 65536; + if( (pgsz & (pgsz-1))!=0 ){ + fprintf(stderr, "invalid page size: %02x%02x\n", aBuf[16], aBuf[17]); + goto endComputeSigs; + } + rewind(in); + for(n=1; (got=fread(aPage, 1, pgsz, in))==pgsz; n++){ + vlogSignature(aPage, pgsz, aBuf); + printf("%4d: %s\n", n, aBuf); + } + +endComputeSigs: + fclose(in); +} + +/* +** Find page signatures for all named files. +*/ +int main(int argc, char **argv){ + int i; + for(i=1; i<argc; i++) computeSigs(argv[i]); + return 0; +} diff --git a/tool/replace.tcl b/tool/replace.tcl new file mode 100644 index 0000000..e87cb92 --- /dev/null +++ b/tool/replace.tcl @@ -0,0 +1,23 @@ +#!/usr/bin/tcl +# +# Replace string with another string -OR- include +# only lines successfully modified with a regular +# expression. +# +fconfigure stdout -translation binary -encoding binary +fconfigure stderr -translation binary -encoding binary +set mode [string tolower [lindex $argv 0]] +set from [lindex $argv 1] +set to [lindex $argv 2] +if {-1 == [lsearch -exact [list exact regsub include] $mode]} {exit 1} +if {[string length $from]==0} {exit 2} +while {![eof stdin]} { + set line [gets stdin] + if {[eof stdin]} break + switch -exact $mode { + exact {set line [string map [list $from $to] $line]} + regsub {regsub -all -- $from $line $to line} + include {if {[regsub -all -- $from $line $to line]==0} continue} + } + puts stdout $line +} diff --git a/tool/restore_jrnl.tcl b/tool/restore_jrnl.tcl new file mode 100644 index 0000000..05af4f9 --- /dev/null +++ b/tool/restore_jrnl.tcl @@ -0,0 +1,233 @@ +# 2010 January 7 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements utility functions for SQLite library. +# +# This file attempts to restore the header of a journal. +# This may be useful for rolling-back the last committed +# transaction from a recovered journal. +# + +package require sqlite3 + +set parm_error 0 +set fix_chksums 0 +set dump_pages 0 +set db_name "" + +for {set i 0} {$i<$argc} {incr i} { + if {[lindex $argv $i] == "-fix_chksums"} { + set fix_chksums -1 + } elseif {[lindex $argv $i] == "-dump_pages"} { + set dump_pages -1 + } elseif {$db_name == ""} { + set db_name [lindex $argv $i] + set jrnl_name $db_name-journal + } else { + set parm_error -1 + } +} +if {$parm_error || $db_name == ""} { + puts "USAGE: restore_jrnl.tcl \[-fix_chksums\] \[-dump_pages\] db_name" + puts "Example: restore_jrnl.tcl foo.sqlite" + return +} + +# is there a way to determine this? +set sectsz 512 + +# Copy file $from into $to +# +proc copy_file {from to} { + file copy -force $from $to +} + +# Execute some SQL +# +proc catchsql {sql} { + set rc [catch {uplevel [list db eval $sql]} msg] + list $rc $msg +} + +# Perform a test +# +proc do_test {name cmd expected} { + puts -nonewline "$name ..." + set res [uplevel $cmd] + if {$res eq $expected} { + puts Ok + } else { + puts Error + puts " Got: $res" + puts " Expected: $expected" + } +} + +# Calc checksum nonce from journal page data. +# +proc calc_nonce {jrnl_pgno} { + global sectsz + global db_pgsz + global jrnl_name + set jrnl_pg_offset [expr $sectsz+((4+$db_pgsz+4)*$jrnl_pgno)] + set nonce [hexio_get_int [hexio_read $jrnl_name [expr $jrnl_pg_offset+4+$db_pgsz] 4]] + for {set i [expr $db_pgsz-200]} {$i>0} {set i [expr $i-200]} { + set byte [hexio_get_int [hexio_read $jrnl_name [expr $jrnl_pg_offset+4+$i] 1]] + set nonce [expr $nonce-$byte] + } + return $nonce +} + +# Calc checksum from journal page data. +# +proc calc_chksum {jrnl_pgno} { + global sectsz + global db_pgsz + global jrnl_name + global nonce + set jrnl_pg_offset [expr $sectsz+((4+$db_pgsz+4)*$jrnl_pgno)] + set chksum $nonce + for {set i [expr $db_pgsz-200]} {$i>0} {set i [expr $i-200]} { + set byte [hexio_get_int [hexio_read $jrnl_name [expr $jrnl_pg_offset+4+$i] 1]] + set chksum [expr $chksum+$byte] + } + return $chksum +} + +# Print journal page data in hex dump form +# +proc dump_jrnl_page {jrnl_pgno} { + global sectsz + global db_pgsz + global jrnl_name + + # print a header block for the page + puts [string repeat "-" 79] + set jrnl_pg_offset [expr $sectsz+((4+$db_pgsz+4)*$jrnl_pgno)] + set db_pgno [hexio_get_int [hexio_read $jrnl_name [expr $jrnl_pg_offset] 4]] + set chksum [hexio_get_int [hexio_read $jrnl_name [expr $jrnl_pg_offset+4+$db_pgsz] 4]] + set nonce [calc_nonce $jrnl_pgno] + puts [ format {jrnl_pg_offset: %08x (%d) jrnl_pgno: %d db_pgno: %d} \
+ $jrnl_pg_offset $jrnl_pg_offset \
+ $jrnl_pgno $db_pgno]
+ puts [ format {nonce: %08x chksum: %08x} \
+ $nonce $chksum]
+ + # now hex dump the data + # This is derived from the Tcler's WIKI
+ set fid [open $jrnl_name r]
+ fconfigure $fid -translation binary -encoding binary
+ seek $fid [expr $jrnl_pg_offset+4]
+ set data [read $fid $db_pgsz]
+ close $fid
+ for {set addr 0} {$addr<$db_pgsz} {set addr [expr $addr+16]} { + # get 16 bytes of data
+ set s [string range $data $addr [expr $addr+16]]
+
+ # Convert the data to hex and to characters.
+ binary scan $s H*@0a* hex ascii
+
+ # Replace non-printing characters in the data.
+ regsub -all -- {[^[:graph:] ]} $ascii {.} ascii
+
+ # Split the 16 bytes into two 8-byte chunks
+ regexp -- {(.{16})(.{0,16})} $hex -> hex1 hex2
+
+ # Convert the hex to pairs of hex digits
+ regsub -all -- {..} $hex1 {& } hex1
+ regsub -all -- {..} $hex2 {& } hex2
+
+ # Print the hex and ascii data
+ puts [ format {%08x %-24s %-24s %-16s} \
+ $addr $hex1 $hex2 $ascii ]
+ }
+} + +# Setup for the tests. Make a backup copy of the files. +# +if [file exist $db_name.org] { + puts "ERROR: during back-up: $db_name.org exists already." + return; +} +if [file exist $jrnl_name.org] { + puts "ERROR: during back-up: $jrnl_name.org exists already." + return +} +copy_file $db_name $db_name.org +copy_file $jrnl_name $jrnl_name.org + +set db_fsize [file size $db_name] +set db_pgsz [hexio_get_int [hexio_read $db_name 16 2]] +set db_npage [expr {$db_fsize / $db_pgsz}] + +set jrnl_fsize [file size $jrnl_name] +set jrnl_npage [expr {($jrnl_fsize - $sectsz) / (4 + $db_pgsz + 4)}] + +# calculate checksum nonce for first page +set nonce [calc_nonce 0] + +# verify all the pages in the journal use the same nonce +for {set i 1} {$i<$jrnl_npage} {incr i} { + set tnonce [calc_nonce $i] + if {$tnonce != $nonce} { + puts "WARNING: different nonces: 0=$nonce $i=$tnonce" + if {$fix_chksums } { + set jrnl_pg_offset [expr $sectsz+((4+$db_pgsz+4)*$i)] + set tchksum [calc_chksum $i] + hexio_write $jrnl_name [expr $jrnl_pg_offset+4+$db_pgsz] [format %08x $tchksum] + puts "INFO: fixing chksum: $i=$tchksum" + } + } +} + +# verify all the page numbers in the journal +for {set i 0} {$i<$jrnl_npage} {incr i} { + set jrnl_pg_offset [expr $sectsz+((4+$db_pgsz+4)*$i)] + set db_pgno [hexio_get_int [hexio_read $jrnl_name $jrnl_pg_offset 4]] + if {$db_pgno < 1} { + puts "WARNING: page number < 1: $i=$db_pgno" + } + if {$db_pgno >= $db_npage} { + puts "WARNING: page number >= $db_npage: $i=$db_pgno" + } +} + +# dump page data +if {$dump_pages} { + for {set i 0} {$i<$jrnl_npage} {incr i} { + dump_jrnl_page $i + } +} + +# write the 8 byte magic string +hexio_write $jrnl_name 0 d9d505f920a163d7 + +# write -1 for number of records +hexio_write $jrnl_name 8 ffffffff + +# write 00 for checksum nonce +hexio_write $jrnl_name 12 [format %08x $nonce] + +# write page count +hexio_write $jrnl_name 16 [format %08x $db_npage] + +# write sector size +hexio_write $jrnl_name 20 [format %08x $sectsz] + +# write page size +hexio_write $jrnl_name 24 [format %08x $db_pgsz] + +# check the integrity of the database with the patched journal +sqlite3 db $db_name +do_test restore_jrnl-1.0 { + catchsql {PRAGMA integrity_check} +} {0 ok} +db close + diff --git a/tool/rollback-test.c b/tool/rollback-test.c new file mode 100644 index 0000000..915d9d2 --- /dev/null +++ b/tool/rollback-test.c @@ -0,0 +1,155 @@ +/* +** This program is used to generate and verify databases with hot journals. +** Use this program to generate a hot journal on one machine and verify +** that it rolls back correctly on another machine with a different +** architecture. +** +** Usage: +** +** rollback-test new [-utf8] [-utf16le] [-utf16be] [-pagesize=N] DATABASE +** rollback-test check DATABASE +** rollback-test crash [-wal] [-rollback] DATABASE +*/ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include "sqlite3.h" + +static void usage(char *argv0){ + fprintf(stderr, + "Usage: %s new [-utf8] [-utf16le] [-utf16be] [-pagesize=N] DATABASE\n" + " %s check DATABASE\n" + " %s crash [-wal] DATABASE\n", + argv0, argv0, argv0 + ); + exit(1); +} + +static sqlite3 *openDb(const char *zFilename){ + int rc; + sqlite3 *db; + rc = sqlite3_open(zFilename, &db); + if( rc ){ + fprintf(stderr, "Cannot open \"%s\": %s\n", + zFilename, sqlite3_errmsg(db)); + sqlite3_close(db); + exit(1); + } + return db; +} + +static int nReply = 0; +static char zReply[1000]; + +static int execCallback(void *NotUsed, int nArg, char **azArg, char **azCol){ + int i, n; + char *z; + for(i=0; i<nArg; i++){ + z = azArg[i]; + if( z==0 ) z = "NULL"; + if( nReply>0 && nReply<sizeof(zReply)-1 ) zReply[nReply++] = ' '; + n = strlen(z); + if( nReply+n>=sizeof(zReply)-1 ) n = sizeof(zReply) - nReply - 1; + memcpy(&zReply[nReply], z, n); + nReply += n; + zReply[nReply] = 0; + } + return 0; +} + +static void runSql(sqlite3 *db, const char *zSql){ + char *zErr = 0; + int rc; + nReply = 0; + rc = sqlite3_exec(db, zSql, execCallback, 0, &zErr); + if( zErr ){ + fprintf(stderr, "SQL error: %s\n", zErr); + exit(1); + } + if( rc ){ + fprintf(stderr, "SQL error: %s\n", sqlite3_errmsg(db)); + exit(1); + } +} + +int main(int argc, char **argv){ + sqlite3 *db; + int i; + + if( argc<3 ) usage(argv[0]); + if( strcmp(argv[1], "new")==0 ){ + db = openDb(argv[argc-1]); + for(i=2; i<argc-1; i++){ + if( strcmp(argv[i],"-utf8")==0 ){ + runSql(db, "PRAGMA encoding=UTF8"); + }else if( strcmp(argv[i], "-utf16le")==0 ){ + runSql(db, "PRAGMA encoding=UTF16LE"); + }else if( strcmp(argv[i], "-utf16be")==0 ){ + runSql(db, "PRAGMA encoding=UTF16BE"); + }else if( strncmp(argv[i], "-pagesize=", 10)==0 ){ + int szPg = atoi(&argv[i][10]); + char zBuf[100]; + sprintf(zBuf, "PRAGMA pagesize=%d", szPg); + runSql(db, zBuf); + }else{ + fprintf(stderr, "unknown option %s\n", argv[i]); + usage(argv[0]); + } + } + runSql(db, + "BEGIN;" + "CREATE TABLE t1(x INTEGER PRIMARY KEY, y);" + "INSERT INTO t1(y) VALUES('abcdefghijklmnopqrstuvwxyz');" + "INSERT INTO t1(y) VALUES('abcdefghijklmnopqrstuvwxyz');" + "INSERT INTO t1(y) SELECT y FROM t1;" /* 4 */ + "INSERT INTO t1(y) SELECT y FROM t1;" /* 8 */ + "INSERT INTO t1(y) SELECT y FROM t1;" /* 16 */ + "INSERT INTO t1(y) SELECT y FROM t1;" /* 32 */ + "INSERT INTO t1(y) SELECT y FROM t1;" /* 64 */ + "INSERT INTO t1(y) SELECT y FROM t1;" /* 128 */ + "INSERT INTO t1(y) SELECT y FROM t1;" /* 256 */ + "INSERT INTO t1(y) SELECT y FROM t1;" /* 512 */ + "INSERT INTO t1(y) SELECT y FROM t1;" /* 1024 */ + "UPDATE t1 SET y=(y || x);" + "CREATE INDEX t1y ON t1(y);" + "COMMIT;" + ); + sqlite3_close(db); + }else if( strcmp(argv[1], "check")==0 ){ + db = openDb(argv[argc-1]); + runSql(db, "PRAGMA integrity_check"); + if( strcmp(zReply, "ok")!=0 ){ + fprintf(stderr, "Integrity check: %s\n", zReply); + exit(1); + } + runSql(db, + "SELECT count(*) FROM t1 WHERE y<>('abcdefghijklmnopqrstuvwxyz' || x)" + ); + if( strcmp(zReply, "0")!=0 ){ + fprintf(stderr, "Wrong content\n"); + exit(1); + } + printf("Ok\n"); + }else if( strcmp(argv[1], "crash")==0 ){ + db = openDb(argv[argc-1]); + for(i=2; i<argc-1; i++){ + if( strcmp(argv[i],"-wal")==0 ){ + runSql(db, "PRAGMA journal_mode=WAL"); + }else if( strcmp(argv[i], "-rollback")==0 ){ + runSql(db, "PRAGMA journal_mode=DELETE"); + }else{ + fprintf(stderr, "unknown option %s\n", argv[i]); + usage(argv[0]); + } + } + runSql(db, + "PRAGMA cache_size=10;" + "BEGIN;" + "UPDATE t1 SET y=(y || -x)" + ); + exit(0); + }else{ + usage(argv[0]); + } + return 0; +} diff --git a/tool/run-speed-test.sh b/tool/run-speed-test.sh new file mode 100644 index 0000000..0e970ea --- /dev/null +++ b/tool/run-speed-test.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# +# This is a template for a script used for day-to-day size and +# performance monitoring of SQLite. Typical usage: +# +# sh run-speed-test.sh trunk # Baseline measurement of trunk +# sh run-speed-test.sh x1 # Measure some experimental change +# fossil test-diff --tk cout-trunk.txt cout-x1.txt # View chanages +# +# There are multiple output files, all with a base name given by +# the first argument: +# +# summary-$BASE.txt # Copy of standard output +# cout-$BASE.txt # cachegrind output +# explain-$BASE.txt # EXPLAIN listings (only with --explain) +# +if test "$1" = "" +then + echo "Usage: $0 OUTPUTFILE [OPTIONS]" + exit +fi +NAME=$1 +shift +CC_OPTS="-DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_MEMSYS5" +SPEEDTEST_OPTS="--shrink-memory --reprepare --heap 10000000 64" +SIZE=5 +doExplain=0 +while test "$1" != ""; do + case $1 in + --reprepare) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" + ;; + --autovacuum) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" + ;; + --utf16be) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" + ;; + --stats) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" + ;; + --without-rowid) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" + ;; + --nomemstat) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" + ;; + --wal) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --journal wal" + ;; + --size) + shift; SIZE=$1 + ;; + --explain) + doExplain=1 + ;; + --heap) + CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_MEMSYS5" + shift; + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --heap $1 64" + ;; + *) + CC_OPTS="$CC_OPTS $1" + ;; + esac + shift +done +SPEEDTEST_OPTS="$SPEEDTEST_OPTS --size $SIZE" +echo "NAME = $NAME" | tee summary-$NAME.txt +echo "SPEEDTEST_OPTS = $SPEEDTEST_OPTS" | tee -a summary-$NAME.txt +echo "CC_OPTS = $CC_OPTS" | tee -a summary-$NAME.txt +rm -f cachegrind.out.* speedtest1 speedtest1.db sqlite3.o +gcc -g -Os -Wall -I. $CC_OPTS -c sqlite3.c +size sqlite3.o | tee -a summary-$NAME.txt +if test $doExplain -eq 1; then + gcc -g -Os -Wall -I. $CC_OPTS \ + -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ + ./shell.c ./sqlite3.c -o sqlite3 -ldl -lpthread +fi +SRC=./speedtest1.c +gcc -g -Os -Wall -I. $CC_OPTS $SRC ./sqlite3.o -o speedtest1 -ldl -lpthread +ls -l speedtest1 | tee -a summary-$NAME.txt +valgrind --tool=cachegrind ./speedtest1 speedtest1.db \ + $SPEEDTEST_OPTS 2>&1 | tee -a summary-$NAME.txt +size sqlite3.o | tee -a summary-$NAME.txt +wc sqlite3.c +cg_anno.tcl cachegrind.out.* >cout-$NAME.txt +if test $doExplain -eq 1; then + ./speedtest1 --explain $SPEEDTEST_OPTS | ./sqlite3 >explain-$NAME.txt +fi diff --git a/tool/showdb.c b/tool/showdb.c new file mode 100644 index 0000000..1b80c7f --- /dev/null +++ b/tool/showdb.c @@ -0,0 +1,1251 @@ +/* +** A utility for printing all or part of an SQLite database file. +*/ +#include <stdio.h> +#include <ctype.h> +#define ISDIGIT(X) isdigit((unsigned char)(X)) +#define ISPRINT(X) isprint((unsigned char)(X)) +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> + +#if !defined(_MSC_VER) +#include <unistd.h> +#else +#include <io.h> +#endif + +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include "sqlite3.h" + +typedef unsigned char u8; /* unsigned 8-bit */ +typedef unsigned int u32; /* unsigned 32-bit */ +typedef sqlite3_int64 i64; /* signed 64-bit */ +typedef sqlite3_uint64 u64; /* unsigned 64-bit */ + + +static struct GlobalData { + u32 pagesize; /* Size of a database page */ + int dbfd; /* File descriptor for reading the DB */ + u32 mxPage; /* Last page number */ + int perLine; /* HEX elements to print per line */ + int bRaw; /* True to access db file via OS APIs */ + sqlite3_file *pFd; /* File descriptor for non-raw mode */ + sqlite3 *pDb; /* Database handle that owns pFd */ +} g = {1024, -1, 0, 16, 0, 0, 0}; + +/* +** Convert the var-int format into i64. Return the number of bytes +** in the var-int. Write the var-int value into *pVal. +*/ +static int decodeVarint(const unsigned char *z, i64 *pVal){ + i64 v = 0; + int i; + for(i=0; i<8; i++){ + v = (v<<7) + (z[i]&0x7f); + if( (z[i]&0x80)==0 ){ *pVal = v; return i+1; } + } + v = (v<<8) + (z[i]&0xff); + *pVal = v; + return 9; +} + +/* +** Extract a big-endian 32-bit integer +*/ +static u32 decodeInt32(const u8 *z){ + return (z[0]<<24) + (z[1]<<16) + (z[2]<<8) + z[3]; +} + +/* Report an out-of-memory error and die. +*/ +static void out_of_memory(void){ + fprintf(stderr,"Out of memory...\n"); + exit(1); +} + +/* +** Open a database connection. +*/ +static sqlite3 *openDatabase(const char *zPrg, const char *zName){ + sqlite3 *db = 0; + int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_URI; + int rc = sqlite3_open_v2(zName, &db, flags, 0); + if( rc!=SQLITE_OK ){ + const char *zErr = sqlite3_errmsg(db); + fprintf(stderr, "%s: can't open %s (%s)\n", zPrg, zName, zErr); + sqlite3_close(db); + exit(1); + } + return db; +} + +/************************************************************************** +** Beginning of low-level file access functions. +** +** All low-level access to the database file read by this program is +** performed using the following four functions: +** +** fileOpen() - open the db file +** fileClose() - close the db file +** fileRead() - read raw data from the db file +** fileGetsize() - return the size of the db file in bytes +*/ + +/* +** Open the database file. +*/ +static void fileOpen(const char *zPrg, const char *zName){ + assert( g.dbfd<0 ); + if( g.bRaw==0 ){ + int rc; + void *pArg = (void *)(&g.pFd); + g.pDb = openDatabase(zPrg, zName); + rc = sqlite3_file_control(g.pDb, "main", SQLITE_FCNTL_FILE_POINTER, pArg); + if( rc!=SQLITE_OK ){ + fprintf(stderr, + "%s: failed to obtain fd for %s (SQLite too old?)\n", zPrg, zName + ); + exit(1); + } + }else{ + g.dbfd = open(zName, O_RDONLY); + if( g.dbfd<0 ){ + fprintf(stderr,"%s: can't open %s\n", zPrg, zName); + exit(1); + } + } +} + +/* +** Close the database file opened by fileOpen() +*/ +static void fileClose(){ + if( g.bRaw==0 ){ + sqlite3_close(g.pDb); + g.pDb = 0; + g.pFd = 0; + }else{ + close(g.dbfd); + g.dbfd = -1; + } +} + +/* +** Read content from the file. +** +** Space to hold the content is obtained from sqlite3_malloc() and needs +** to be freed by the caller. +*/ +static unsigned char *fileRead(sqlite3_int64 ofst, int nByte){ + unsigned char *aData; + int got; + aData = sqlite3_malloc64(32+(i64)nByte); + if( aData==0 ) out_of_memory(); + memset(aData, 0, nByte+32); + if( g.bRaw==0 ){ + int rc = g.pFd->pMethods->xRead(g.pFd, (void*)aData, nByte, ofst); + if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){ + fprintf(stderr, "error in xRead() - %d\n", rc); + exit(1); + } + }else{ + lseek(g.dbfd, (long)ofst, SEEK_SET); + got = read(g.dbfd, aData, nByte); + if( got>0 && got<nByte ) memset(aData+got, 0, nByte-got); + } + return aData; +} + +/* +** Return the size of the file in byte. +*/ +static i64 fileGetsize(void){ + i64 res = 0; + if( g.bRaw==0 ){ + int rc = g.pFd->pMethods->xFileSize(g.pFd, &res); + if( rc!=SQLITE_OK ){ + fprintf(stderr, "error in xFileSize() - %d\n", rc); + exit(1); + } + }else{ + struct stat sbuf; + fstat(g.dbfd, &sbuf); + res = (sqlite3_int64)(sbuf.st_size); + } + return res; +} + +/* +** End of low-level file access functions. +**************************************************************************/ + +/* +** Print a range of bytes as hex and as ascii. +*/ +static unsigned char *print_byte_range( + sqlite3_int64 ofst, /* First byte in the range of bytes to print */ + int nByte, /* Number of bytes to print */ + int printOfst /* Add this amount to the index on the left column */ +){ + unsigned char *aData; + int i, j; + const char *zOfstFmt; + + if( ((printOfst+nByte)&~0xfff)==0 ){ + zOfstFmt = " %03x: "; + }else if( ((printOfst+nByte)&~0xffff)==0 ){ + zOfstFmt = " %04x: "; + }else if( ((printOfst+nByte)&~0xfffff)==0 ){ + zOfstFmt = " %05x: "; + }else if( ((printOfst+nByte)&~0xffffff)==0 ){ + zOfstFmt = " %06x: "; + }else{ + zOfstFmt = " %08x: "; + } + + aData = fileRead(ofst, nByte); + for(i=0; i<nByte; i += g.perLine){ + int go = 0; + for(j=0; j<g.perLine; j++){ + if( i+j>nByte ){ break; } + if( aData[i+j] ){ go = 1; break; } + } + if( !go && i>0 && i+g.perLine<nByte ) continue; + fprintf(stdout, zOfstFmt, i+printOfst); + for(j=0; j<g.perLine; j++){ + if( i+j>nByte ){ + fprintf(stdout, " "); + }else{ + fprintf(stdout,"%02x ", aData[i+j]); + } + } + for(j=0; j<g.perLine; j++){ + if( i+j>nByte ){ + fprintf(stdout, " "); + }else{ + fprintf(stdout,"%c", ISPRINT(aData[i+j]) ? aData[i+j] : '.'); + } + } + fprintf(stdout,"\n"); + } + return aData; +} + +/* +** Print an entire page of content as hex +*/ +static void print_page(u32 iPg){ + i64 iStart; + unsigned char *aData; + iStart = ((i64)(iPg-1))*g.pagesize; + fprintf(stdout, "Page %u: (offsets 0x%llx..0x%llx)\n", + iPg, iStart, iStart+g.pagesize-1); + aData = print_byte_range(iStart, g.pagesize, 0); + sqlite3_free(aData); +} + + +/* Print a line of decoded output showing a 4-byte unsigned integer. +*/ +static void print_decode_line( + unsigned char *aData, /* Content being decoded */ + int ofst, int nByte, /* Start and size of decode */ + const char *zMsg /* Message to append */ +){ + int i, j; + u32 val = aData[ofst]; + char zBuf[100]; + sprintf(zBuf, " %03x: %02x", ofst, aData[ofst]); + i = (int)strlen(zBuf); + for(j=1; j<4; j++){ + if( j>=nByte ){ + sprintf(&zBuf[i], " "); + }else{ + sprintf(&zBuf[i], " %02x", aData[ofst+j]); + val = val*256 + aData[ofst+j]; + } + i += (int)strlen(&zBuf[i]); + } + sprintf(&zBuf[i], " %10u", val); + printf("%s %s\n", zBuf, zMsg); +} + +/* +** Decode the database header. +*/ +static void print_db_header(void){ + unsigned char *aData; + aData = print_byte_range(0, 100, 0); + printf("Decoded:\n"); + print_decode_line(aData, 16, 2, "Database page size"); + print_decode_line(aData, 18, 1, "File format write version"); + print_decode_line(aData, 19, 1, "File format read version"); + print_decode_line(aData, 20, 1, "Reserved space at end of page"); + print_decode_line(aData, 24, 4, "File change counter"); + print_decode_line(aData, 28, 4, "Size of database in pages"); + print_decode_line(aData, 32, 4, "Page number of first freelist page"); + print_decode_line(aData, 36, 4, "Number of freelist pages"); + print_decode_line(aData, 40, 4, "Schema cookie"); + print_decode_line(aData, 44, 4, "Schema format version"); + print_decode_line(aData, 48, 4, "Default page cache size"); + print_decode_line(aData, 52, 4, "Largest auto-vac root page"); + print_decode_line(aData, 56, 4, "Text encoding"); + print_decode_line(aData, 60, 4, "User version"); + print_decode_line(aData, 64, 4, "Incremental-vacuum mode"); + print_decode_line(aData, 68, 4, "Application ID"); + print_decode_line(aData, 72, 4, "meta[8]"); + print_decode_line(aData, 76, 4, "meta[9]"); + print_decode_line(aData, 80, 4, "meta[10]"); + print_decode_line(aData, 84, 4, "meta[11]"); + print_decode_line(aData, 88, 4, "meta[12]"); + print_decode_line(aData, 92, 4, "Change counter for version number"); + print_decode_line(aData, 96, 4, "SQLite version number"); + sqlite3_free(aData); +} + +/* +** Describe cell content. +*/ +static i64 describeContent( + unsigned char *a, /* Cell content */ + i64 nLocal, /* Bytes in a[] */ + char *zDesc /* Write description here */ +){ + i64 nDesc = 0; + int n, j; + i64 i, x, v; + const unsigned char *pData; + const unsigned char *pLimit; + char sep = ' '; + + pLimit = &a[nLocal]; + n = decodeVarint(a, &x); + pData = &a[x]; + a += n; + i = x - n; + while( i>0 && pData<=pLimit ){ + n = decodeVarint(a, &x); + a += n; + i -= n; + nLocal -= n; + zDesc[0] = sep; + sep = ','; + nDesc++; + zDesc++; + if( x==0 ){ + sprintf(zDesc, "*"); /* NULL is a "*" */ + }else if( x>=1 && x<=6 ){ + v = (signed char)pData[0]; + pData++; + switch( x ){ + case 6: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; + case 5: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; + case 4: v = (v<<8) + pData[0]; pData++; + case 3: v = (v<<8) + pData[0]; pData++; + case 2: v = (v<<8) + pData[0]; pData++; + } + sprintf(zDesc, "%lld", v); + }else if( x==7 ){ + sprintf(zDesc, "real"); + pData += 8; + }else if( x==8 ){ + sprintf(zDesc, "0"); + }else if( x==9 ){ + sprintf(zDesc, "1"); + }else if( x>=12 ){ + i64 size = (x-12)/2; + if( (x&1)==0 ){ + sprintf(zDesc, "blob(%lld)", size); + }else{ + sprintf(zDesc, "txt(%lld)", size); + } + pData += size; + } + j = (int)strlen(zDesc); + zDesc += j; + nDesc += j; + } + return nDesc; +} + +/* +** Compute the local payload size given the total payload size and +** the page size. +*/ +static i64 localPayload(i64 nPayload, char cType){ + i64 maxLocal; + i64 minLocal; + i64 surplus; + i64 nLocal; + if( cType==13 ){ + /* Table leaf */ + maxLocal = g.pagesize-35; + minLocal = (g.pagesize-12)*32/255-23; + }else{ + maxLocal = (g.pagesize-12)*64/255-23; + minLocal = (g.pagesize-12)*32/255-23; + } + if( nPayload>maxLocal ){ + surplus = minLocal + (nPayload-minLocal)%(g.pagesize-4); + if( surplus<=maxLocal ){ + nLocal = surplus; + }else{ + nLocal = minLocal; + } + }else{ + nLocal = nPayload; + } + return nLocal; +} + + +/* +** Create a description for a single cell. +** +** The return value is the local cell size. +*/ +static i64 describeCell( + unsigned char cType, /* Page type */ + unsigned char *a, /* Cell content */ + int showCellContent, /* Show cell content if true */ + char **pzDesc /* Store description here */ +){ + int i; + i64 nDesc = 0; + int n = 0; + u32 leftChild; + i64 nPayload; + i64 rowid; + i64 nLocal; + static char zDesc[1000]; + i = 0; + if( cType<=5 ){ + leftChild = ((a[0]*256 + a[1])*256 + a[2])*256 + a[3]; + a += 4; + n += 4; + sprintf(zDesc, "lx: %u ", leftChild); + nDesc = strlen(zDesc); + } + if( cType!=5 ){ + i = decodeVarint(a, &nPayload); + a += i; + n += i; + sprintf(&zDesc[nDesc], "n: %lld ", nPayload); + nDesc += strlen(&zDesc[nDesc]); + nLocal = localPayload(nPayload, cType); + }else{ + nPayload = nLocal = 0; + } + if( cType==5 || cType==13 ){ + i = decodeVarint(a, &rowid); + a += i; + n += i; + sprintf(&zDesc[nDesc], "r: %lld ", rowid); + nDesc += strlen(&zDesc[nDesc]); + } + if( nLocal<nPayload ){ + u32 ovfl; + unsigned char *b = &a[nLocal]; + ovfl = ((b[0]*256 + b[1])*256 + b[2])*256 + b[3]; + sprintf(&zDesc[nDesc], "ov: %u ", ovfl); + nDesc += strlen(&zDesc[nDesc]); + n += 4; + } + if( showCellContent && cType!=5 ){ + nDesc += describeContent(a, nLocal, &zDesc[nDesc-1]); + } + *pzDesc = zDesc; + return nLocal+n; +} + +/* Print an offset followed by nByte bytes. Add extra white-space +** at the end so that subsequent text is aligned. +*/ +static void printBytes( + unsigned char *aData, /* Content being decoded */ + unsigned char *aStart, /* Start of content to be printed */ + int nByte /* Number of bytes to print */ +){ + int j; + printf(" %03x: ", (int)(aStart-aData)); + for(j=0; j<9; j++){ + if( j>=nByte ){ + printf(" "); + }else{ + printf("%02x ", aStart[j]); + } + } +} + + +/* +** Write a full decode on stdout for the cell at a[ofst]. +** Assume the page contains a header of size szPgHdr bytes. +*/ +static void decodeCell( + unsigned char *a, /* Page content (without the page-1 header) */ + unsigned pgno, /* Page number */ + int iCell, /* Cell index */ + int szPgHdr, /* Size of the page header. 0 or 100 */ + int ofst /* Cell begins at a[ofst] */ +){ + int i, j = 0; + u32 leftChild; + i64 k; + i64 nPayload; + i64 rowid; + i64 nHdr; + i64 iType; + i64 nLocal; + unsigned char *x = a + ofst; + unsigned char *end; + unsigned char cType = a[0]; + int nCol = 0; + int szCol[2000]; + int ofstCol[2000]; + int typeCol[2000]; + + printf("Cell[%d]:\n", iCell); + if( cType<=5 ){ + leftChild = ((x[0]*256 + x[1])*256 + x[2])*256 + x[3]; + printBytes(a, x, 4); + printf("left child page:: %u\n", leftChild); + x += 4; + } + if( cType!=5 ){ + i = decodeVarint(x, &nPayload); + printBytes(a, x, i); + nLocal = localPayload(nPayload, cType); + if( nLocal==nPayload ){ + printf("payload-size: %lld\n", nPayload); + }else{ + printf("payload-size: %lld (%lld local, %lld overflow)\n", + nPayload, nLocal, nPayload-nLocal); + } + x += i; + }else{ + nPayload = nLocal = 0; + } + end = x + nLocal; + if( cType==5 || cType==13 ){ + i = decodeVarint(x, &rowid); + printBytes(a, x, i); + printf("rowid: %lld\n", rowid); + x += i; + } + if( nLocal>0 ){ + i = decodeVarint(x, &nHdr); + printBytes(a, x, i); + printf("record-header-size: %d\n", (int)nHdr); + j = i; + nCol = 0; + k = nHdr; + while( x+j<=end && j<nHdr ){ + const char *zTypeName; + int sz = 0; + char zNm[30]; + i = decodeVarint(x+j, &iType); + printBytes(a, x+j, i); + printf("typecode[%d]: %d - ", nCol, (int)iType); + switch( iType ){ + case 0: zTypeName = "NULL"; sz = 0; break; + case 1: zTypeName = "int8"; sz = 1; break; + case 2: zTypeName = "int16"; sz = 2; break; + case 3: zTypeName = "int24"; sz = 3; break; + case 4: zTypeName = "int32"; sz = 4; break; + case 5: zTypeName = "int48"; sz = 6; break; + case 6: zTypeName = "int64"; sz = 8; break; + case 7: zTypeName = "double"; sz = 8; break; + case 8: zTypeName = "zero"; sz = 0; break; + case 9: zTypeName = "one"; sz = 0; break; + case 10: + case 11: zTypeName = "error"; sz = 0; break; + default: { + sz = (int)(iType-12)/2; + sprintf(zNm, (iType&1)==0 ? "blob(%d)" : "text(%d)", sz); + zTypeName = zNm; + break; + } + } + printf("%s\n", zTypeName); + szCol[nCol] = sz; + ofstCol[nCol] = (int)k; + typeCol[nCol] = (int)iType; + k += sz; + nCol++; + j += i; + } + for(i=0; i<nCol && ofstCol[i]+szCol[i]<=nLocal; i++){ + int s = ofstCol[i]; + i64 v; + const unsigned char *pData; + if( szCol[i]==0 ) continue; + printBytes(a, x+s, szCol[i]); + printf("data[%d]: ", i); + pData = x+s; + if( typeCol[i]<=7 ){ + v = (signed char)pData[0]; + for(k=1; k<szCol[i]; k++){ + v = (v<<8) + pData[k]; + } + if( typeCol[i]==7 ){ + double r; + memcpy(&r, &v, sizeof(r)); + printf("%#g\n", r); + }else{ + printf("%lld\n", v); + } + }else{ + int ii, jj; + char zConst[32]; + if( (typeCol[i]&1)==0 ){ + zConst[0] = 'x'; + zConst[1] = '\''; + for(ii=2, jj=0; jj<szCol[i] && ii<24; jj++, ii+=2){ + sprintf(zConst+ii, "%02x", pData[jj]); + } + }else{ + zConst[0] = '\''; + for(ii=1, jj=0; jj<szCol[i] && ii<24; jj++, ii++){ + zConst[ii] = ISPRINT(pData[jj]) ? pData[jj] : '.'; + } + zConst[ii] = 0; + } + if( jj<szCol[i] ){ + memcpy(zConst+ii, "...'", 5); + }else{ + memcpy(zConst+ii, "'", 2); + } + printf("%s\n", zConst); + } + j = ofstCol[i] + szCol[i]; + } + } + if( j<nLocal ){ + printBytes(a, x+j, 0); + printf("... %lld bytes of content ...\n", nLocal-j); + } + if( nLocal<nPayload ){ + printBytes(a, x+nLocal, 4); + printf("overflow-page: %u\n", decodeInt32(x+nLocal)); + } +} + + +/* +** Decode a btree page +*/ +static void decode_btree_page( + unsigned char *a, /* Page content */ + int pgno, /* Page number */ + int hdrSize, /* Size of the page header. 0 or 100 */ + char *zArgs /* Flags to control formatting */ +){ + const char *zType = "unknown"; + int nCell; + int i, j; + int iCellPtr; + int showCellContent = 0; + int showMap = 0; + int cellToDecode = -2; + char *zMap = 0; + switch( a[0] ){ + case 2: zType = "index interior node"; break; + case 5: zType = "table interior node"; break; + case 10: zType = "index leaf"; break; + case 13: zType = "table leaf"; break; + } + while( zArgs[0] ){ + switch( zArgs[0] ){ + case 'c': showCellContent = 1; break; + case 'm': showMap = 1; break; + case 'd': { + if( !ISDIGIT(zArgs[1]) ){ + cellToDecode = -1; + }else{ + cellToDecode = 0; + while( ISDIGIT(zArgs[1]) ){ + zArgs++; + cellToDecode = cellToDecode*10 + zArgs[0] - '0'; + } + } + break; + } + } + zArgs++; + } + nCell = a[3]*256 + a[4]; + iCellPtr = (a[0]==2 || a[0]==5) ? 12 : 8; + if( cellToDecode>=nCell ){ + printf("Page %d has only %d cells\n", pgno, nCell); + return; + } + printf("Header on btree page %d:\n", pgno); + print_decode_line(a, 0, 1, zType); + print_decode_line(a, 1, 2, "Offset to first freeblock"); + print_decode_line(a, 3, 2, "Number of cells on this page"); + print_decode_line(a, 5, 2, "Offset to cell content area"); + print_decode_line(a, 7, 1, "Fragmented byte count"); + if( a[0]==2 || a[0]==5 ){ + print_decode_line(a, 8, 4, "Right child"); + } + if( cellToDecode==(-2) && nCell>0 ){ + printf(" key: lx=left-child n=payload-size r=rowid\n"); + } + if( showMap ){ + zMap = sqlite3_malloc(g.pagesize); + memset(zMap, '.', g.pagesize); + memset(zMap, '1', hdrSize); + memset(&zMap[hdrSize], 'H', iCellPtr); + memset(&zMap[hdrSize+iCellPtr], 'P', 2*nCell); + } + for(i=0; i<nCell; i++){ + int cofst = iCellPtr + i*2; + char *zDesc; + i64 n; + + cofst = a[cofst]*256 + a[cofst+1]; + n = describeCell(a[0], &a[cofst-hdrSize], showCellContent, &zDesc); + if( showMap ){ + char zBuf[30]; + memset(&zMap[cofst], '*', (size_t)n); + zMap[cofst] = '['; + zMap[cofst+n-1] = ']'; + sprintf(zBuf, "%d", i); + j = (int)strlen(zBuf); + if( j<=n-2 ) memcpy(&zMap[cofst+1], zBuf, j); + } + if( cellToDecode==(-2) ){ + printf(" %03x: cell[%d] %s\n", cofst, i, zDesc); + }else if( cellToDecode==(-1) || cellToDecode==i ){ + decodeCell(a, pgno, i, hdrSize, cofst-hdrSize); + } + } + if( showMap ){ + printf("Page map: (H=header P=cell-index 1=page-1-header .=free-space)\n"); + for(i=0; (u32)i<g.pagesize; i+=64){ + printf(" %03x: %.64s\n", i, &zMap[i]); + } + sqlite3_free(zMap); + } +} + +/* +** Decode a freelist trunk page. +*/ +static void decode_trunk_page( + u32 pgno, /* The page number */ + int detail, /* Show leaf pages if true */ + int recursive /* Follow the trunk change if true */ +){ + u32 i; + u32 n; + unsigned char *a; + while( pgno>0 ){ + a = fileRead((pgno-1)*g.pagesize, g.pagesize); + printf("Decode of freelist trunk page %d:\n", pgno); + print_decode_line(a, 0, 4, "Next freelist trunk page"); + print_decode_line(a, 4, 4, "Number of entries on this page"); + if( detail ){ + n = decodeInt32(&a[4]); + for(i=0; i<n && i<g.pagesize/4; i++){ + u32 x = decodeInt32(&a[8+4*i]); + char zIdx[13]; + sprintf(zIdx, "[%d]", i); + printf(" %5s %7u", zIdx, x); + if( i%5==4 ) printf("\n"); + } + if( i%5!=0 ) printf("\n"); + } + if( !recursive ){ + pgno = 0; + }else{ + pgno = decodeInt32(&a[0]); + } + sqlite3_free(a); + } +} + +/* +** A short text comment on the use of each page. +*/ +static char **zPageUse; + +/* +** Add a comment on the use of a page. +*/ +static void page_usage_msg(u32 pgno, const char *zFormat, ...){ + va_list ap; + char *zMsg; + + va_start(ap, zFormat); + zMsg = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + if( pgno<=0 || pgno>g.mxPage ){ + printf("ERROR: page %d out of range 1..%u: %s\n", + pgno, g.mxPage, zMsg); + sqlite3_free(zMsg); + return; + } + if( zPageUse[pgno]!=0 ){ + printf("ERROR: page %d used multiple times:\n", pgno); + printf("ERROR: previous: %s\n", zPageUse[pgno]); + printf("ERROR: current: %s\n", zMsg); + sqlite3_free(zPageUse[pgno]); + } + zPageUse[pgno] = zMsg; +} + +/* +** Find overflow pages of a cell and describe their usage. +*/ +static void page_usage_cell( + unsigned char cType, /* Page type */ + unsigned char *a, /* Cell content */ + u32 pgno, /* page containing the cell */ + int cellno /* Index of the cell on the page */ +){ + int i; + int n = 0; + i64 nPayload; + i64 rowid; + i64 nLocal; + i = 0; + if( cType<=5 ){ + a += 4; + n += 4; + } + if( cType!=5 ){ + i = decodeVarint(a, &nPayload); + a += i; + n += i; + nLocal = localPayload(nPayload, cType); + }else{ + nPayload = nLocal = 0; + } + if( cType==5 || cType==13 ){ + i = decodeVarint(a, &rowid); + a += i; + n += i; + } + if( nLocal<nPayload ){ + u32 ovfl = decodeInt32(a+nLocal); + u32 cnt = 0; + while( ovfl && (cnt++)<g.mxPage ){ + page_usage_msg(ovfl, "overflow %d from cell %d of page %u", + cnt, cellno, pgno); + a = fileRead((ovfl-1)*(sqlite3_int64)g.pagesize, 4); + ovfl = decodeInt32(a); + sqlite3_free(a); + } + } +} + +/* +** True if the memory is all zeros +*/ +static int allZero(unsigned char *a, int n){ + while( n && (a++)[0]==0 ){ n--; } + return n==0; +} + + +/* +** Describe the usages of a b-tree page. +** +** If parent==0, then this is the root of a btree. If parent<0 then +** this is an orphan page. +*/ +static void page_usage_btree( + u32 pgno, /* Page to describe */ + int parent, /* Parent of this page. 0 for root pages */ + int idx, /* Which child of the parent */ + const char *zName /* Name of the table */ +){ + unsigned char *a; + const char *zType = "corrupt node"; + int nCell; + int i; + int hdr = pgno==1 ? 100 : 0; + char zEntry[30]; + + if( pgno<=0 || pgno>g.mxPage ) return; + a = fileRead((pgno-1)*g.pagesize, g.pagesize); + switch( a[hdr] ){ + case 0: { + if( allZero(a, g.pagesize) ){ + zType = "zeroed page"; + }else if( parent<0 ){ + return; + }else{ + zType = "corrupt node"; + } + break; + } + case 2: zType = "interior node of index"; break; + case 5: zType = "interior node of table"; break; + case 10: zType = "leaf of index"; break; + case 13: zType = "leaf of table"; break; + default: { + if( parent<0 ) return; + zType = "corrupt node"; + } + } + nCell = a[hdr+3]*256 + a[hdr+4]; + if( nCell==1 ){ + sqlite3_snprintf(sizeof(zEntry),zEntry,"1 row"); + }else{ + sqlite3_snprintf(sizeof(zEntry),zEntry,"%d rows", nCell); + } + if( parent>0 ){ + page_usage_msg(pgno, "%s [%s], child %d of page %d, %s", + zType, zName, idx, parent, zEntry); + }else if( parent==0 ){ + page_usage_msg(pgno, "root %s [%s], %s", zType, zName, zEntry); + }else{ + page_usage_msg(pgno, "orphaned %s, %s", zType, zEntry); + } + if( a[hdr]==2 || a[hdr]==5 ){ + int cellstart = hdr+12; + u32 child; + for(i=0; i<nCell; i++){ + u32 cellidx; + u32 ofst; + + cellidx = cellstart + i*2; + if( cellidx+1 >= g.pagesize ){ + printf("ERROR: page %d too many cells (%d)\n", pgno, nCell); + break; + } + ofst = a[cellidx]*256 + a[cellidx+1]; + if( ofst<cellidx+2 || ofst+4>=g.pagesize ){ + printf("ERROR: page %d cell %d out of bounds\n", pgno, i); + continue; + } + child = decodeInt32(a+ofst); + page_usage_btree(child, pgno, i, zName); + } + child = decodeInt32(a+cellstart-4); + page_usage_btree(child, pgno, i, zName); + } + if( a[hdr]==2 || a[hdr]==10 || a[hdr]==13 ){ + int cellstart = hdr + 8 + 4*(a[hdr]<=5); + for(i=0; i<nCell; i++){ + int ofst; + ofst = cellstart + i*2; + ofst = a[ofst]*256 + a[ofst+1]; + page_usage_cell(a[hdr], a+ofst, pgno, i); + } + } + sqlite3_free(a); +} + +/* +** Determine page usage by the freelist +*/ +static void page_usage_freelist(u32 pgno){ + unsigned char *a; + int cnt = 0; + int i; + int n; + int iNext; + int parent = 1; + + while( pgno>0 && pgno<=g.mxPage && (u32)(cnt++)<g.mxPage ){ + page_usage_msg(pgno, "freelist trunk #%d child of %d", cnt, parent); + a = fileRead((pgno-1)*g.pagesize, g.pagesize); + iNext = decodeInt32(a); + n = decodeInt32(a+4); + if( n>(g.pagesize - 8)/4 ){ + printf("ERROR: page %d too many freelist entries (%d)\n", pgno, n); + n = (g.pagesize - 8)/4; + } + for(i=0; i<n; i++){ + int child = decodeInt32(a + (i*4+8)); + page_usage_msg(child, "freelist leaf, child %d of trunk page %d", + i, pgno); + } + sqlite3_free(a); + parent = pgno; + pgno = iNext; + } +} + +/* +** Determine pages used as PTRMAP pages +*/ +static void page_usage_ptrmap(u8 *a){ + if( decodeInt32(a+52) ){ + int usable = g.pagesize - a[20]; + u64 pgno = 2; + int perPage = usable/5; + while( pgno<=g.mxPage ){ + page_usage_msg((u32)pgno, "PTRMAP page covering %llu..%llu", + pgno+1, pgno+perPage); + pgno += perPage + 1; + } + } +} + +/* +** Try to figure out how every page in the database file is being used. +*/ +static void page_usage_report(const char *zPrg, const char *zDbName){ + u32 i, j; + int rc; + sqlite3 *db; + sqlite3_stmt *pStmt; + unsigned char *a; + char zQuery[200]; + + /* Avoid the pathological case */ + if( g.mxPage<1 ){ + printf("empty database\n"); + return; + } + + /* Open the database file */ + db = openDatabase(zPrg, zDbName); + + /* Set up global variables zPageUse[] and g.mxPage to record page + ** usages */ + zPageUse = sqlite3_malloc64( sizeof(zPageUse[0])*(g.mxPage+1) ); + if( zPageUse==0 ) out_of_memory(); + memset(zPageUse, 0, sizeof(zPageUse[0])*(g.mxPage+1)); + + /* Discover the usage of each page */ + a = fileRead(0, 100); + page_usage_freelist(decodeInt32(a+32)); + page_usage_ptrmap(a); + sqlite3_free(a); + page_usage_btree(1, 0, 0, "sqlite_schema"); + sqlite3_exec(db, "PRAGMA writable_schema=ON", 0, 0, 0); + for(j=0; j<2; j++){ + sqlite3_snprintf(sizeof(zQuery), zQuery, + "SELECT type, name, rootpage FROM SQLITE_MASTER WHERE rootpage" + " ORDER BY rowid %s", j?"DESC":""); + rc = sqlite3_prepare_v2(db, zQuery, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + u32 pgno = (u32)sqlite3_column_int64(pStmt, 2); + page_usage_btree(pgno, 0, 0, (const char*)sqlite3_column_text(pStmt,1)); + } + }else{ + printf("ERROR: cannot query database: %s\n", sqlite3_errmsg(db)); + } + rc = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) break; + } + sqlite3_close(db); + + /* Print the report and free memory used */ + for(i=1; i<=g.mxPage; i++){ + if( zPageUse[i]==0 ) page_usage_btree(i, -1, 0, 0); + printf("%5u: %s\n", i, zPageUse[i] ? zPageUse[i] : "???"); + } + for(i=1; i<=g.mxPage; i++){ + sqlite3_free(zPageUse[i]); + } + sqlite3_free(zPageUse); + zPageUse = 0; +} + +/* +** Try to figure out how every page in the database file is being used. +*/ +static void ptrmap_coverage_report(const char *zDbName){ + u64 pgno; + unsigned char *aHdr; + unsigned char *a; + int usable; + int perPage; + int i; + + /* Avoid the pathological case */ + if( g.mxPage<1 ){ + printf("empty database\n"); + return; + } + + /* Make sure PTRMAPs are used in this database */ + aHdr = fileRead(0, 100); + if( aHdr[55]==0 ){ + printf("database does not use PTRMAP pages\n"); + return; + } + usable = g.pagesize - aHdr[20]; + perPage = usable/5; + sqlite3_free(aHdr); + printf("%5d: root of sqlite_schema\n", 1); + for(pgno=2; pgno<=g.mxPage; pgno += perPage+1){ + printf("%5llu: PTRMAP page covering %llu..%llu\n", pgno, + pgno+1, pgno+perPage); + a = fileRead((pgno-1)*g.pagesize, usable); + for(i=0; i+5<=usable && pgno+1+i/5<=g.mxPage; i+=5){ + const char *zType = "???"; + u32 iFrom = decodeInt32(&a[i+1]); + switch( a[i] ){ + case 1: zType = "b-tree root page"; break; + case 2: zType = "freelist page"; break; + case 3: zType = "first page of overflow"; break; + case 4: zType = "later page of overflow"; break; + case 5: zType = "b-tree non-root page"; break; + } + printf("%5llu: %s, parent=%u\n", pgno+1+i/5, zType, iFrom); + } + sqlite3_free(a); + } +} + +/* +** Check the range validity for a page number. Print an error and +** exit if the page is out of range. +*/ +static void checkPageValidity(int iPage){ + if( iPage<1 || iPage>g.mxPage ){ + fprintf(stderr, "Invalid page number %d: valid range is 1..%d\n", + iPage, g.mxPage); + exit(1); + } +} + +/* +** Print a usage comment +*/ +static void usage(const char *argv0){ + fprintf(stderr, "Usage %s ?--uri? FILENAME ?args...?\n\n", argv0); + fprintf(stderr, + "switches:\n" + " --raw Read db file directly, bypassing SQLite VFS\n" + "args:\n" + " dbheader Show database header\n" + " pgidx Index of how each page is used\n" + " ptrmap Show all PTRMAP page content\n" + " NNN..MMM Show hex of pages NNN through MMM\n" + " NNN..end Show hex of pages NNN through end of file\n" + " NNNb Decode btree page NNN\n" + " NNNbc Decode btree page NNN and show content\n" + " NNNbm Decode btree page NNN and show a layout map\n" + " NNNbdCCC Decode cell CCC on btree page NNN\n" + " NNNt Decode freelist trunk page NNN\n" + " NNNtd Show leaf freelist pages on the decode\n" + " NNNtr Recursively decode freelist starting at NNN\n" + ); +} + +int main(int argc, char **argv){ + sqlite3_int64 szFile; + unsigned char *zPgSz; + const char *zPrg = argv[0]; /* Name of this executable */ + char **azArg = argv; + int nArg = argc; + + /* Check for the "--uri" or "-uri" switch. */ + if( nArg>1 ){ + if( sqlite3_stricmp("-raw", azArg[1])==0 + || sqlite3_stricmp("--raw", azArg[1])==0 + ){ + g.bRaw = 1; + azArg++; + nArg--; + } + } + + if( nArg<2 ){ + usage(zPrg); + exit(1); + } + + fileOpen(zPrg, azArg[1]); + szFile = fileGetsize(); + + zPgSz = fileRead(16, 2); + g.pagesize = zPgSz[0]*256 + zPgSz[1]*65536; + if( g.pagesize==0 ) g.pagesize = 1024; + sqlite3_free(zPgSz); + + printf("Pagesize: %d\n", g.pagesize); + g.mxPage = (u32)((szFile+g.pagesize-1)/g.pagesize); + + printf("Available pages: 1..%u\n", g.mxPage); + if( nArg==2 ){ + u32 i; + for(i=1; i<=g.mxPage; i++) print_page(i); + }else{ + int i; + for(i=2; i<nArg; i++){ + u32 iStart, iEnd; + char *zLeft; + if( strcmp(azArg[i], "dbheader")==0 ){ + print_db_header(); + continue; + } + if( strcmp(azArg[i], "pgidx")==0 ){ + page_usage_report(zPrg, azArg[1]); + continue; + } + if( strcmp(azArg[i], "ptrmap")==0 ){ + ptrmap_coverage_report(azArg[1]); + continue; + } + if( strcmp(azArg[i], "help")==0 ){ + usage(zPrg); + continue; + } + if( !ISDIGIT(azArg[i][0]) ){ + fprintf(stderr, "%s: unknown option: [%s]\n", zPrg, azArg[i]); + continue; + } + iStart = strtoul(azArg[i], &zLeft, 0); + checkPageValidity(iStart); + if( zLeft && strcmp(zLeft,"..end")==0 ){ + iEnd = g.mxPage; + }else if( zLeft && zLeft[0]=='.' && zLeft[1]=='.' ){ + iEnd = strtol(&zLeft[2], 0, 0); + checkPageValidity(iEnd); + }else if( zLeft && zLeft[0]=='b' ){ + int ofst, nByte, hdrSize; + unsigned char *a; + if( iStart==1 ){ + ofst = hdrSize = 100; + nByte = g.pagesize-100; + }else{ + hdrSize = 0; + ofst = (iStart-1)*g.pagesize; + nByte = g.pagesize; + } + a = fileRead(ofst, nByte); + decode_btree_page(a, iStart, hdrSize, &zLeft[1]); + sqlite3_free(a); + continue; + }else if( zLeft && zLeft[0]=='t' ){ + int detail = 0; + int recursive = 0; + int j; + for(j=1; zLeft[j]; j++){ + if( zLeft[j]=='r' ) recursive = 1; + if( zLeft[j]=='d' ) detail = 1; + } + decode_trunk_page(iStart, detail, recursive); + continue; + }else{ + iEnd = iStart; + } + if( iStart<1 || iEnd<iStart || iEnd>g.mxPage ){ + fprintf(stderr, + "Page argument should be LOWER?..UPPER?. Range 1 to %d\n", + g.mxPage); + exit(1); + } + while( iStart<=iEnd ){ + print_page(iStart); + iStart++; + } + } + } + fileClose(); + return 0; +} diff --git a/tool/showjournal.c b/tool/showjournal.c new file mode 100644 index 0000000..51c89da --- /dev/null +++ b/tool/showjournal.c @@ -0,0 +1,138 @@ +/* +** A utility for printing an SQLite database journal. +*/ +#include <stdio.h> +#include <ctype.h> +#include <stdlib.h> +#include <string.h> + +/* +** state information +*/ +static int pageSize = 1024; +static int sectorSize = 512; +static FILE *db = 0; +static int fileSize = 0; +static unsigned cksumNonce = 0; + +/* Report a memory allocation error */ +static void out_of_memory(void){ + fprintf(stderr,"Out of memory...\n"); + exit(1); +} + +/* +** Read N bytes of memory starting at iOfst into space obtained +** from malloc(). +*/ +static unsigned char *read_content(int N, int iOfst){ + int got; + unsigned char *pBuf = malloc(N); + if( pBuf==0 ) out_of_memory(); + fseek(db, iOfst, SEEK_SET); + got = (int)fread(pBuf, 1, N, db); + if( got<0 ){ + fprintf(stderr, "I/O error reading %d bytes from %d\n", N, iOfst); + memset(pBuf, 0, N); + }else if( got<N ){ + fprintf(stderr, "Short read: got only %d of %d bytes from %d\n", + got, N, iOfst); + memset(&pBuf[got], 0, N-got); + } + return pBuf; +} + +/* Print a line of decode output showing a 4-byte integer. +*/ +static unsigned print_decode_line( + const unsigned char *aData, /* Content being decoded */ + int ofst, int nByte, /* Start and size of decode */ + const char *zMsg /* Message to append */ +){ + int i, j; + unsigned val = aData[ofst]; + char zBuf[100]; + sprintf(zBuf, " %05x: %02x", ofst, aData[ofst]); + i = (int)strlen(zBuf); + for(j=1; j<4; j++){ + if( j>=nByte ){ + sprintf(&zBuf[i], " "); + }else{ + sprintf(&zBuf[i], " %02x", aData[ofst+j]); + val = val*256 + aData[ofst+j]; + } + i += (int)strlen(&zBuf[i]); + } + sprintf(&zBuf[i], " %10u", val); + printf("%s %s\n", zBuf, zMsg); + return val; +} + +/* +** Read and print a journal header. Store key information (page size, etc) +** in global variables. +*/ +static unsigned decode_journal_header(int iOfst){ + unsigned char *pHdr = read_content(64, iOfst); + unsigned nPage; + printf("Header at offset %d:\n", iOfst); + print_decode_line(pHdr, 0, 4, "Header part 1 (3654616569)"); + print_decode_line(pHdr, 4, 4, "Header part 2 (547447767)"); + nPage = + print_decode_line(pHdr, 8, 4, "page count"); + cksumNonce = + print_decode_line(pHdr, 12, 4, "chksum nonce"); + print_decode_line(pHdr, 16, 4, "initial database size in pages"); + sectorSize = + print_decode_line(pHdr, 20, 4, "sector size"); + pageSize = + print_decode_line(pHdr, 24, 4, "page size"); + print_decode_line(pHdr, 28, 4, "zero"); + print_decode_line(pHdr, 32, 4, "zero"); + print_decode_line(pHdr, 36, 4, "zero"); + print_decode_line(pHdr, 40, 4, "zero"); + free(pHdr); + return nPage; +} + +static void print_page(int iOfst){ + unsigned char *aData; + char zTitle[50]; + aData = read_content(pageSize+8, iOfst); + sprintf(zTitle, "page number for page at offset %d", iOfst); + print_decode_line(aData-iOfst, iOfst, 4, zTitle); + free(aData); +} + +int main(int argc, char **argv){ + int nPage, cnt; + int iOfst; + if( argc!=2 ){ + fprintf(stderr,"Usage: %s FILENAME\n", argv[0]); + exit(1); + } + db = fopen(argv[1], "rb"); + if( db==0 ){ + fprintf(stderr,"%s: can't open %s\n", argv[0], argv[1]); + exit(1); + } + fseek(db, 0, SEEK_END); + fileSize = ftell(db); + printf("journal file size: %d bytes\n", fileSize); + fseek(db, 0, SEEK_SET); + iOfst = 0; + while( iOfst<fileSize ){ + cnt = nPage = (int)decode_journal_header(iOfst); + if( cnt==0 ){ + cnt = (fileSize - sectorSize)/(pageSize+8); + } + iOfst += sectorSize; + while( cnt && iOfst<fileSize ){ + print_page(iOfst); + iOfst += pageSize+8; + } + iOfst = (iOfst/sectorSize + 1)*sectorSize; + } + fclose(db); + return 0; +} diff --git a/tool/showlocks.c b/tool/showlocks.c new file mode 100644 index 0000000..4159a71 --- /dev/null +++ b/tool/showlocks.c @@ -0,0 +1,99 @@ +/* +** This file implements a simple command-line utility that shows all of the +** Posix Advisory Locks on a file. +** +** Usage: +** +** showlocks FILENAME +** +** To compile: gcc -o showlocks showlocks.c +*/ +#include <stdio.h> +#include <unistd.h> +#include <fcntl.h> +#include <stdlib.h> +#include <string.h> + +/* This utility only looks for locks in the first 2 billion bytes */ +#define MX_LCK 2147483647 + +/* +** Print all locks on the inode of "fd" that occur in between +** lwr and upr, inclusive. +*/ +static int showLocksInRange(int fd, off_t lwr, off_t upr){ + int cnt = 0; + struct flock x; + struct lockRange { + off_t lwr; + off_t upr; + } *aPending = 0; + int nAlloc = 1; + int nPending = 0; + int nDone = 0; + + nPending = 1; + aPending = malloc( sizeof(aPending[0]) ); + if( aPending==0 ){ + fprintf(stderr, "out of memory\n"); + exit(1); + } + aPending[0].lwr = lwr; + aPending[0].upr = upr; + + for(nDone=0; nDone<nPending; nDone++){ + lwr = aPending[nDone].lwr; + upr = aPending[nDone].upr; + if( lwr>=upr ) continue; + x.l_type = F_WRLCK; + x.l_whence = SEEK_SET; + x.l_start = lwr; + x.l_len = upr - lwr; + fcntl(fd, F_GETLK, &x); + if( x.l_type==F_UNLCK ) continue; + printf("start: %-12d len: %-5d pid: %-5d type: %s\n", + (int)x.l_start, (int)x.l_len, + x.l_pid, x.l_type==F_WRLCK ? "WRLCK" : "RDLCK"); + cnt++; + if( nPending+2 > nAlloc ){ + nAlloc = nAlloc*2 + 2; + aPending = realloc(aPending, sizeof(aPending[0])*nAlloc ); + } + if( aPending==0 ){ + fprintf(stderr, "unable to realloc for %d bytes\n", + (int)sizeof(aPending[0])*(nPending+2)); + exit(1); + } + if( lwr<x.l_start ){ + aPending[nPending].lwr = lwr; + aPending[nPending].upr = x.l_start; + nPending++; + } + if( x.l_start+x.l_len<=upr ){ + aPending[nPending].lwr = x.l_start + x.l_len; + aPending[nPending].upr = upr; + nPending++; + } + } + free(aPending); + return cnt; +} + +int main(int argc, char **argv){ + int fd; + int cnt; + + if( argc!=2 ){ + fprintf(stderr, "Usage: %s FILENAME\n", argv[0]); + return 1; + } + fd = open(argv[1], O_RDWR, 0); + if( fd<0 ){ + fprintf(stderr, "%s: cannot open %s\n", argv[0], argv[1]); + return 1; + } + cnt = showLocksInRange(fd, 0, MX_LCK); + if( cnt==0 ) printf("no locks\n"); + close(fd); + return 0; +} diff --git a/tool/showshm.c b/tool/showshm.c new file mode 100644 index 0000000..03e0fc4 --- /dev/null +++ b/tool/showshm.c @@ -0,0 +1,158 @@ +/* +** A utility for printing content from the wal-index or "shm" file. +*/ +#include <stdio.h> +#include <ctype.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <assert.h> + +#define ISDIGIT(X) isdigit((unsigned char)(X)) +#define ISPRINT(X) isprint((unsigned char)(X)) + +#if !defined(_MSC_VER) +#include <unistd.h> +#include <sys/types.h> +#else +#include <io.h> +#endif + +#include <stdlib.h> +#include <string.h> + +static int fd = -1; /* The open SHM file */ + +/* Report an out-of-memory error and die. +*/ +static void out_of_memory(void){ + fprintf(stderr,"Out of memory...\n"); + exit(1); +} + +/* +** Read content from the file. +** +** Space to hold the content is obtained from malloc() and needs to be +** freed by the caller. +*/ +static unsigned char *getContent(int ofst, int nByte){ + unsigned char *aData; + aData = malloc(nByte); + if( aData==0 ) out_of_memory(); + lseek(fd, ofst, SEEK_SET); + read(fd, aData, nByte); + return aData; +} + +/* +** Flags values +*/ +#define FG_HEX 1 /* Show as hex */ +#define FG_NBO 2 /* Native byte order */ +#define FG_PGSZ 4 /* Show as page-size */ + +/* Print a line of decode output showing a 4-byte integer. +*/ +static void print_decode_line( + unsigned char *aData, /* Content being decoded */ + int ofst, int nByte, /* Start and size of decode */ + unsigned flg, /* Display flags */ + const char *zMsg /* Message to append */ +){ + int i, j; + int val = aData[ofst]; + char zBuf[100]; + sprintf(zBuf, " %03x: %02x", ofst, aData[ofst]); + i = (int)strlen(zBuf); + for(j=1; j<4; j++){ + if( j>=nByte ){ + sprintf(&zBuf[i], " "); + }else{ + sprintf(&zBuf[i], " %02x", aData[ofst+j]); + val = val*256 + aData[ofst+j]; + } + i += (int)strlen(&zBuf[i]); + } + if( nByte==8 ){ + for(j=4; j<8; j++){ + sprintf(&zBuf[i], " %02x", aData[ofst+j]); + i += (int)strlen(&zBuf[i]); + } + } + if( flg & FG_NBO ){ + assert( nByte==4 ); + memcpy(&val, aData+ofst, 4); + } + sprintf(&zBuf[i], " "); + i += 12; + if( flg & FG_PGSZ ){ + unsigned short sz; + memcpy(&sz, aData+ofst, 2); + sprintf(&zBuf[i], " %9d", sz==1 ? 65536 : sz); + }else if( flg & FG_HEX ){ + sprintf(&zBuf[i], " 0x%08x", val); + }else if( nByte<8 ){ + sprintf(&zBuf[i], " %9d", val); + } + printf("%s %s\n", zBuf, zMsg); +} + +/* +** Print an instance of the WalIndexHdr object. ix is either 0 or 1 +** to select which header to print. +*/ +static void print_index_hdr(unsigned char *aData, int ix){ + int i; + assert( ix==0 || ix==1 ); + i = ix ? 48 : 0; + print_decode_line(aData, 0+i, 4, FG_NBO, "Wal-index version"); + print_decode_line(aData, 4+i, 4, 0, "unused padding"); + print_decode_line(aData, 8+i, 4, FG_NBO, "transaction counter"); + print_decode_line(aData,12+i, 1, 0, "1 when initialized"); + print_decode_line(aData,13+i, 1, 0, "true if WAL cksums are bigendian"); + print_decode_line(aData,14+i, 2, FG_PGSZ, "database page size"); + print_decode_line(aData,16+i, 4, FG_NBO, "mxFrame"); + print_decode_line(aData,20+i, 4, FG_NBO, "Size of database in pages"); + print_decode_line(aData,24+i, 8, 0, "Cksum of last frame in -wal"); + print_decode_line(aData,32+i, 8, 0, "Salt values from the -wal"); + print_decode_line(aData,40+i, 8, 0, "Cksum over all prior fields"); +} + +/* +** Print the WalCkptInfo object +*/ +static void print_ckpt_info(unsigned char *aData){ + const int i = 96; + int j; + print_decode_line(aData, 0+i, 4, FG_NBO, "nBackfill"); + for(j=0; j<5; j++){ + char zLabel[100]; + sprintf(zLabel, "aReadMark[%d]", j); + print_decode_line(aData, 4*j+4+i, 4, FG_NBO, zLabel); + } + print_decode_line(aData,24+i, 8, 0, "aLock"); + print_decode_line(aData,32+i, 4, FG_NBO, "nBackfillAttempted"); + print_decode_line(aData,36+i, 4, FG_NBO, "notUsed0"); +} + + +int main(int argc, char **argv){ + unsigned char *aData; + if( argc<2 ){ + fprintf(stderr,"Usage: %s FILENAME\n", argv[0]); + exit(1); + } + fd = open(argv[1], O_RDONLY); + if( fd<0 ){ + fprintf(stderr,"%s: can't open %s\n", argv[0], argv[1]); + exit(1); + } + aData = getContent(0, 136); + print_index_hdr(aData, 0); + print_index_hdr(aData, 1); + print_ckpt_info(aData); + free(aData); + close(fd); + return 0; +} diff --git a/tool/showstat4.c b/tool/showstat4.c new file mode 100644 index 0000000..b8a12ad --- /dev/null +++ b/tool/showstat4.c @@ -0,0 +1,163 @@ +/* +** This utility program decodes and displays the content of the +** sqlite_stat4 table in the database file named on the command +** line. +*/ +#include <stdio.h> +#include <string.h> +#include <stdlib.h> +#include <ctype.h> +#include "sqlite3.h" + +#define ISPRINT(X) isprint((unsigned char)(X)) + +typedef sqlite3_int64 i64; /* 64-bit signed integer type */ + + +/* +** Convert the var-int format into i64. Return the number of bytes +** in the var-int. Write the var-int value into *pVal. +*/ +static int decodeVarint(const unsigned char *z, i64 *pVal){ + i64 v = 0; + int i; + for(i=0; i<8; i++){ + v = (v<<7) + (z[i]&0x7f); + if( (z[i]&0x80)==0 ){ *pVal = v; return i+1; } + } + v = (v<<8) + (z[i]&0xff); + *pVal = v; + return 9; +} + + + +int main(int argc, char **argv){ + sqlite3 *db; + sqlite3_stmt *pStmt; + char *zIdx = 0; + int rc, j, x, y, mxHdr; + const unsigned char *aSample; + int nSample; + i64 iVal; + const char *zSep; + int iRow = 0; + + if( argc!=2 ){ + fprintf(stderr, "Usage: %s DATABASE-FILE\n", argv[0]); + exit(1); + } + rc = sqlite3_open(argv[1], &db); + if( rc!=SQLITE_OK || db==0 ){ + fprintf(stderr, "Cannot open database file [%s]\n", argv[1]); + exit(1); + } + rc = sqlite3_prepare_v2(db, + "SELECT tbl||'.'||idx, nEq, nLT, nDLt, sample " + "FROM sqlite_stat4 ORDER BY 1", -1, + &pStmt, 0); + if( rc!=SQLITE_OK || pStmt==0 ){ + fprintf(stderr, "%s\n", sqlite3_errmsg(db)); + sqlite3_close(db); + exit(1); + } + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( zIdx==0 || strcmp(zIdx, (const char*)sqlite3_column_text(pStmt,0))!=0 ){ + if( zIdx ) printf("\n**************************************" + "**************\n\n"); + sqlite3_free(zIdx); + zIdx = sqlite3_mprintf("%s", sqlite3_column_text(pStmt,0)); + iRow = 0; + } + printf("%s sample %d ------------------------------------\n", zIdx, ++iRow); + printf(" nEq = %s\n", sqlite3_column_text(pStmt,1)); + printf(" nLt = %s\n", sqlite3_column_text(pStmt,2)); + printf(" nDLt = %s\n", sqlite3_column_text(pStmt,3)); + printf(" sample = x'"); + aSample = sqlite3_column_blob(pStmt,4); + nSample = sqlite3_column_bytes(pStmt,4); + for(j=0; j<nSample; j++) printf("%02x", aSample[j]); + printf("'\n "); + zSep = " "; + x = decodeVarint(aSample, &iVal); + if( iVal<x || iVal>nSample ){ + printf(" <error>\n"); + continue; + } + y = mxHdr = (int)iVal; + while( x<mxHdr ){ + int sz; + i64 v; + x += decodeVarint(aSample+x, &iVal); + if( x>mxHdr ) break; + if( iVal<0 ) break; + switch( iVal ){ + case 0: sz = 0; break; + case 1: sz = 1; break; + case 2: sz = 2; break; + case 3: sz = 3; break; + case 4: sz = 4; break; + case 5: sz = 6; break; + case 6: sz = 8; break; + case 7: sz = 8; break; + case 8: sz = 0; break; + case 9: sz = 0; break; + case 10: + case 11: sz = 0; break; + default: sz = (int)(iVal-12)/2; break; + } + if( y+sz>nSample ) break; + if( iVal==0 ){ + printf("%sNULL", zSep); + }else if( iVal==8 || iVal==9 ){ + printf("%s%d", zSep, ((int)iVal)-8); + }else if( iVal<=7 ){ + v = (signed char)aSample[y]; + for(j=1; j<sz; j++){ + v = (v<<8) + aSample[y+j]; + } + if( iVal==7 ){ + double r; + char *z; + memcpy(&r, &v, sizeof(r)); + z = sqlite3_mprintf("%s%!.15g", zSep, r); + printf("%s", z); + sqlite3_free(z); + }else{ + printf("%s%lld", zSep, v); + } + }else if( (iVal&1)==0 ){ + printf("%sx'", zSep); + for(j=0; j<sz; j++){ + printf("%02x", aSample[y+j]); + } + printf("'"); + }else{ + printf("%s'", zSep); + for(j=0; j<sz; j++){ + char c = (char)aSample[y+j]; + if( ISPRINT(c) ){ + if( c=='\'' || c=='\\' ) putchar('\\'); + putchar(c); + }else if( c=='\n' ){ + printf("\\n"); + }else if( c=='\t' ){ + printf("\\t"); + }else if( c=='\r' ){ + printf("\\r"); + }else{ + printf("\\%03o", c); + } + } + printf("'"); + } + zSep = ","; + y += sz; + } + printf("\n"); + } + sqlite3_free(zIdx); + sqlite3_finalize(pStmt); + sqlite3_close(db); + return 0; +} diff --git a/tool/showwal.c b/tool/showwal.c new file mode 100644 index 0000000..7e6c0e1 --- /dev/null +++ b/tool/showwal.c @@ -0,0 +1,639 @@ +/* +** A utility for printing content from a write-ahead log file. +*/ +#include <stdio.h> +#include <ctype.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> + +#define ISDIGIT(X) isdigit((unsigned char)(X)) +#define ISPRINT(X) isprint((unsigned char)(X)) + +#if !defined(_MSC_VER) +#include <unistd.h> +#include <sys/types.h> +#else +#include <io.h> +#endif + +#include <stdlib.h> +#include <string.h> + + +static int pagesize = 1024; /* Size of a database page */ +static int fd = -1; /* File descriptor for reading the WAL file */ +static int mxFrame = 0; /* Last frame */ +static int perLine = 16; /* HEX elements to print per line */ + +typedef long long int i64; /* Datatype for 64-bit integers */ + +/* Information for computing the checksum */ +typedef struct Cksum Cksum; +struct Cksum { + int bSwap; /* True to do byte swapping on 32-bit words */ + unsigned s0, s1; /* Current checksum value */ +}; + +/* +** extract a 32-bit big-endian integer +*/ +static unsigned int getInt32(const unsigned char *a){ + unsigned int x = (a[0]<<24) + (a[1]<<16) + (a[2]<<8) + a[3]; + return x; +} + +/* +** Swap bytes on a 32-bit unsigned integer +*/ +static unsigned int swab32(unsigned int x){ + return (((x)&0x000000FF)<<24) + (((x)&0x0000FF00)<<8) + + (((x)&0x00FF0000)>>8) + (((x)&0xFF000000)>>24); +} + +/* Extend the checksum. Reinitialize the checksum if bInit is true. +*/ +static void extendCksum( + Cksum *pCksum, + unsigned char *aData, + unsigned int nByte, + int bInit +){ + unsigned int *a32; + if( bInit ){ + int a = 0; + *((char*)&a) = 1; + if( a==1 ){ + /* Host is little-endian */ + pCksum->bSwap = getInt32(aData)!=0x377f0682; + }else{ + /* Host is big-endian */ + pCksum->bSwap = getInt32(aData)!=0x377f0683; + } + pCksum->s0 = 0; + pCksum->s1 = 0; + } + a32 = (unsigned int*)aData; + while( nByte>0 ){ + unsigned int x0 = a32[0]; + unsigned int x1 = a32[1]; + if( pCksum->bSwap ){ + x0 = swab32(x0); + x1 = swab32(x1); + } + pCksum->s0 += x0 + pCksum->s1; + pCksum->s1 += x1 + pCksum->s0; + nByte -= 8; + a32 += 2; + } +} + +/* +** Convert the var-int format into i64. Return the number of bytes +** in the var-int. Write the var-int value into *pVal. +*/ +static int decodeVarint(const unsigned char *z, i64 *pVal){ + i64 v = 0; + int i; + for(i=0; i<8; i++){ + v = (v<<7) + (z[i]&0x7f); + if( (z[i]&0x80)==0 ){ *pVal = v; return i+1; } + } + v = (v<<8) + (z[i]&0xff); + *pVal = v; + return 9; +} + +/* Report an out-of-memory error and die. +*/ +static void out_of_memory(void){ + fprintf(stderr,"Out of memory...\n"); + exit(1); +} + +/* +** Read content from the file. +** +** Space to hold the content is obtained from malloc() and needs to be +** freed by the caller. +*/ +static unsigned char *getContent(i64 ofst, int nByte){ + unsigned char *aData; + aData = malloc(nByte); + if( aData==0 ) out_of_memory(); + lseek(fd, ofst, SEEK_SET); + read(fd, aData, nByte); + return aData; +} + +/* +** Print a range of bytes as hex and as ascii. +*/ +static void print_byte_range( + int ofst, /* First byte in the range of bytes to print */ + int nByte, /* Number of bytes to print */ + unsigned char *aData, /* Content to print */ + int printOfst /* Add this amount to the index on the left column */ +){ + int i, j; + const char *zOfstFmt; + + if( ((printOfst+nByte)&~0xfff)==0 ){ + zOfstFmt = " %03x: "; + }else if( ((printOfst+nByte)&~0xffff)==0 ){ + zOfstFmt = " %04x: "; + }else if( ((printOfst+nByte)&~0xfffff)==0 ){ + zOfstFmt = " %05x: "; + }else if( ((printOfst+nByte)&~0xffffff)==0 ){ + zOfstFmt = " %06x: "; + }else{ + zOfstFmt = " %08x: "; + } + + for(i=0; i<nByte; i += perLine){ + fprintf(stdout, zOfstFmt, i+printOfst); + for(j=0; j<perLine; j++){ + if( i+j>nByte ){ + fprintf(stdout, " "); + }else{ + fprintf(stdout,"%02x ", aData[i+j]); + } + } + for(j=0; j<perLine; j++){ + if( i+j>nByte ){ + fprintf(stdout, " "); + }else{ + fprintf(stdout,"%c", ISPRINT(aData[i+j]) ? aData[i+j] : '.'); + } + } + fprintf(stdout,"\n"); + } +} + +/* Print a line of decode output showing a 4-byte integer. +*/ +static void print_decode_line( + unsigned char *aData, /* Content being decoded */ + int ofst, int nByte, /* Start and size of decode */ + int asHex, /* If true, output value as hex */ + const char *zMsg /* Message to append */ +){ + int i, j; + int val = aData[ofst]; + char zBuf[100]; + sprintf(zBuf, " %03x: %02x", ofst, aData[ofst]); + i = (int)strlen(zBuf); + for(j=1; j<4; j++){ + if( j>=nByte ){ + sprintf(&zBuf[i], " "); + }else{ + sprintf(&zBuf[i], " %02x", aData[ofst+j]); + val = val*256 + aData[ofst+j]; + } + i += (int)strlen(&zBuf[i]); + } + if( asHex ){ + sprintf(&zBuf[i], " 0x%08x", val); + }else{ + sprintf(&zBuf[i], " %9d", val); + } + printf("%s %s\n", zBuf, zMsg); +} + +/* +** Print an entire page of content as hex +*/ +static void print_frame(int iFrame){ + i64 iStart; + unsigned char *aData; + iStart = 32 + (i64)(iFrame-1)*(pagesize+24); + fprintf(stdout, "Frame %d: (offsets 0x%llx..0x%llx)\n", + iFrame, iStart, iStart+pagesize+24); + aData = getContent(iStart, pagesize+24); + print_decode_line(aData, 0, 4, 0, "Page number"); + print_decode_line(aData, 4, 4, 0, "DB size, or 0 for non-commit"); + print_decode_line(aData, 8, 4, 1, "Salt-1"); + print_decode_line(aData,12, 4, 1, "Salt-2"); + print_decode_line(aData,16, 4, 1, "Checksum-1"); + print_decode_line(aData,20, 4, 1, "Checksum-2"); + print_byte_range(iStart+24, pagesize, aData+24, 0); + free(aData); +} + +/* +** Summarize a single frame on a single line. +*/ +static void print_oneline_frame(int iFrame, Cksum *pCksum){ + i64 iStart; + unsigned char *aData; + unsigned int s0, s1; + iStart = 32 + (i64)(iFrame-1)*(pagesize+24); + aData = getContent(iStart, 24); + extendCksum(pCksum, aData, 8, 0); + extendCksum(pCksum, getContent(iStart+24, pagesize), pagesize, 0); + s0 = getInt32(aData+16); + s1 = getInt32(aData+20); + fprintf(stdout, "Frame %4d: %6d %6d 0x%08x,%08x 0x%08x,%08x", + iFrame, + getInt32(aData), + getInt32(aData+4), + getInt32(aData+8), + getInt32(aData+12), + s0, + s1 + ); + if( s0==pCksum->s0 && s1==pCksum->s1 ){ + fprintf(stdout, "\n"); + }else{ + fprintf(stdout, " should be 0x%08x,%08x\n", + pCksum->s0, pCksum->s1); + } + + /* Reset the checksum so that a single frame checksum failure will not + ** cause all subsequent frames to also show a failure. */ + pCksum->s0 = s0; + pCksum->s1 = s1; + free(aData); +} + +/* +** Decode the WAL header. +*/ +static void print_wal_header(Cksum *pCksum){ + unsigned char *aData; + aData = getContent(0, 32); + if( pCksum ){ + extendCksum(pCksum, aData, 24, 1); + printf("Checksum byte order: %s\n", pCksum->bSwap ? "swapped" : "native"); + } + printf("WAL Header:\n"); + print_decode_line(aData, 0, 4,1,"Magic. 0x377f0682 (le) or 0x377f0683 (be)"); + print_decode_line(aData, 4, 4, 0, "File format"); + print_decode_line(aData, 8, 4, 0, "Database page size"); + print_decode_line(aData, 12,4, 0, "Checkpoint sequence number"); + print_decode_line(aData, 16,4, 1, "Salt-1"); + print_decode_line(aData, 20,4, 1, "Salt-2"); + print_decode_line(aData, 24,4, 1, "Checksum-1"); + print_decode_line(aData, 28,4, 1, "Checksum-2"); + if( pCksum ){ + if( pCksum->s0!=getInt32(aData+24) ){ + printf("**** cksum-1 mismatch: 0x%08x\n", pCksum->s0); + } + if( pCksum->s1!=getInt32(aData+28) ){ + printf("**** cksum-2 mismatch: 0x%08x\n", pCksum->s1); + } + } + free(aData); +} +/* +** Describe cell content. +*/ +static i64 describeContent( + unsigned char *a, /* Cell content */ + i64 nLocal, /* Bytes in a[] */ + char *zDesc /* Write description here */ +){ + int nDesc = 0; + int n, j; + i64 i, x, v; + const unsigned char *pData; + const unsigned char *pLimit; + char sep = ' '; + + pLimit = &a[nLocal]; + n = decodeVarint(a, &x); + pData = &a[x]; + a += n; + i = x - n; + while( i>0 && pData<=pLimit ){ + n = decodeVarint(a, &x); + a += n; + i -= n; + nLocal -= n; + zDesc[0] = sep; + sep = ','; + nDesc++; + zDesc++; + if( x==0 ){ + sprintf(zDesc, "*"); /* NULL is a "*" */ + }else if( x>=1 && x<=6 ){ + v = (signed char)pData[0]; + pData++; + switch( x ){ + case 6: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; + case 5: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; + case 4: v = (v<<8) + pData[0]; pData++; + case 3: v = (v<<8) + pData[0]; pData++; + case 2: v = (v<<8) + pData[0]; pData++; + } + sprintf(zDesc, "%lld", v); + }else if( x==7 ){ + sprintf(zDesc, "real"); + pData += 8; + }else if( x==8 ){ + sprintf(zDesc, "0"); + }else if( x==9 ){ + sprintf(zDesc, "1"); + }else if( x>=12 ){ + i64 size = (x-12)/2; + if( (x&1)==0 ){ + sprintf(zDesc, "blob(%lld)", size); + }else{ + sprintf(zDesc, "txt(%lld)", size); + } + pData += size; + } + j = (int)strlen(zDesc); + zDesc += j; + nDesc += j; + } + return nDesc; +} + +/* +** Compute the local payload size given the total payload size and +** the page size. +*/ +static i64 localPayload(i64 nPayload, char cType){ + i64 maxLocal; + i64 minLocal; + i64 surplus; + i64 nLocal; + if( cType==13 ){ + /* Table leaf */ + maxLocal = pagesize-35; + minLocal = (pagesize-12)*32/255-23; + }else{ + maxLocal = (pagesize-12)*64/255-23; + minLocal = (pagesize-12)*32/255-23; + } + if( nPayload>maxLocal ){ + surplus = minLocal + (nPayload-minLocal)%(pagesize-4); + if( surplus<=maxLocal ){ + nLocal = surplus; + }else{ + nLocal = minLocal; + } + }else{ + nLocal = nPayload; + } + return nLocal; +} + +/* +** Create a description for a single cell. +** +** The return value is the local cell size. +*/ +static i64 describeCell( + unsigned char cType, /* Page type */ + unsigned char *a, /* Cell content */ + int showCellContent, /* Show cell content if true */ + char **pzDesc /* Store description here */ +){ + int i; + i64 nDesc = 0; + int n = 0; + int leftChild; + i64 nPayload; + i64 rowid; + i64 nLocal; + static char zDesc[1000]; + i = 0; + if( cType<=5 ){ + leftChild = ((a[0]*256 + a[1])*256 + a[2])*256 + a[3]; + a += 4; + n += 4; + sprintf(zDesc, "lx: %d ", leftChild); + nDesc = strlen(zDesc); + } + if( cType!=5 ){ + i = decodeVarint(a, &nPayload); + a += i; + n += i; + sprintf(&zDesc[nDesc], "n: %lld ", nPayload); + nDesc += strlen(&zDesc[nDesc]); + nLocal = localPayload(nPayload, cType); + }else{ + nPayload = nLocal = 0; + } + if( cType==5 || cType==13 ){ + i = decodeVarint(a, &rowid); + a += i; + n += i; + sprintf(&zDesc[nDesc], "r: %lld ", rowid); + nDesc += strlen(&zDesc[nDesc]); + } + if( nLocal<nPayload ){ + int ovfl; + unsigned char *b = &a[nLocal]; + ovfl = ((b[0]*256 + b[1])*256 + b[2])*256 + b[3]; + sprintf(&zDesc[nDesc], "ov: %d ", ovfl); + nDesc += strlen(&zDesc[nDesc]); + n += 4; + } + if( showCellContent && cType!=5 ){ + nDesc += describeContent(a, nLocal, &zDesc[nDesc-1]); + } + *pzDesc = zDesc; + return nLocal+n; +} + +/* +** Decode a btree page +*/ +static void decode_btree_page( + unsigned char *a, /* Content of the btree page to be decoded */ + int pgno, /* Page number */ + int hdrSize, /* Size of the page1-header in bytes */ + const char *zArgs /* Flags to control formatting */ +){ + const char *zType = "unknown"; + int nCell; + int i, j; + int iCellPtr; + int showCellContent = 0; + int showMap = 0; + char *zMap = 0; + switch( a[0] ){ + case 2: zType = "index interior node"; break; + case 5: zType = "table interior node"; break; + case 10: zType = "index leaf"; break; + case 13: zType = "table leaf"; break; + } + while( zArgs[0] ){ + switch( zArgs[0] ){ + case 'c': showCellContent = 1; break; + case 'm': showMap = 1; break; + } + zArgs++; + } + printf("Decode of btree page %d:\n", pgno); + print_decode_line(a, 0, 1, 0, zType); + print_decode_line(a, 1, 2, 0, "Offset to first freeblock"); + print_decode_line(a, 3, 2, 0, "Number of cells on this page"); + nCell = a[3]*256 + a[4]; + print_decode_line(a, 5, 2, 0, "Offset to cell content area"); + print_decode_line(a, 7, 1, 0, "Fragmented byte count"); + if( a[0]==2 || a[0]==5 ){ + print_decode_line(a, 8, 4, 0, "Right child"); + iCellPtr = 12; + }else{ + iCellPtr = 8; + } + if( nCell>0 ){ + printf(" key: lx=left-child n=payload-size r=rowid\n"); + } + if( showMap ){ + zMap = malloc(pagesize); + memset(zMap, '.', pagesize); + memset(zMap, '1', hdrSize); + memset(&zMap[hdrSize], 'H', iCellPtr); + memset(&zMap[hdrSize+iCellPtr], 'P', 2*nCell); + } + for(i=0; i<nCell; i++){ + int cofst = iCellPtr + i*2; + char *zDesc; + i64 n; + + cofst = a[cofst]*256 + a[cofst+1]; + n = describeCell(a[0], &a[cofst-hdrSize], showCellContent, &zDesc); + if( showMap ){ + char zBuf[30]; + memset(&zMap[cofst], '*', (size_t)n); + zMap[cofst] = '['; + zMap[cofst+n-1] = ']'; + sprintf(zBuf, "%d", i); + j = (int)strlen(zBuf); + if( j<=n-2 ) memcpy(&zMap[cofst+1], zBuf, j); + } + printf(" %03x: cell[%d] %s\n", cofst, i, zDesc); + } + if( showMap ){ + for(i=0; i<pagesize; i+=64){ + printf(" %03x: %.64s\n", i, &zMap[i]); + } + free(zMap); + } +} + +/* +** Check the range validity for a page number. Print an error and +** exit if the page is out of range. +*/ +static void checkPageValidity(int iPage, int mxPage){ + if( iPage<1 || iPage>mxPage ){ + fprintf(stderr, "Invalid page number %d: valid range is 1..%d\n", + iPage, mxPage); + exit(1); + } +} + +int main(int argc, char **argv){ + struct stat sbuf; + unsigned char zPgSz[4]; + if( argc<2 ){ + fprintf(stderr,"Usage: %s FILENAME ?PAGE? ...\n", argv[0]); + exit(1); + } + fd = open(argv[1], O_RDONLY); + if( fd<0 ){ + fprintf(stderr,"%s: can't open %s\n", argv[0], argv[1]); + exit(1); + } + zPgSz[0] = 0; + zPgSz[1] = 0; + fstat(fd, &sbuf); + if( sbuf.st_size<32 ){ + printf("%s: file too small to be a WAL - only %d bytes\n", + argv[1], (int)sbuf.st_size); + return 0; + } + if( lseek(fd, 8, SEEK_SET)!=8 ){ + printf("\"%s\" seems to not be a valid WAL file\n", argv[1]); + return 1; + } + if( read(fd, zPgSz, 4)!=4 ){ + printf("\"%s\": cannot read the page size\n", argv[1]); + return 1; + } + pagesize = zPgSz[1]*65536 + zPgSz[2]*256 + zPgSz[3]; + if( pagesize==0 ) pagesize = 1024; + printf("Pagesize: %d\n", pagesize); + if( (pagesize & (pagesize-1))!=0 || pagesize<512 || pagesize>65536 ){ + printf("\"%s\": invalid page size.\n", argv[1]); + return 1; + } + mxFrame = (sbuf.st_size - 32)/(pagesize + 24); + printf("Available pages: 1..%d\n", mxFrame); + if( argc==2 ){ + int i; + Cksum x; + print_wal_header(&x); + for(i=1; i<=mxFrame; i++){ + print_oneline_frame(i, &x); + } + }else{ + int i; + for(i=2; i<argc; i++){ + int iStart, iEnd; + char *zLeft; + if( strcmp(argv[i], "header")==0 ){ + print_wal_header(0); + continue; + } + if( !ISDIGIT(argv[i][0]) ){ + fprintf(stderr, "%s: unknown option: [%s]\n", argv[0], argv[i]); + continue; + } + iStart = strtol(argv[i], &zLeft, 0); + checkPageValidity(iStart, mxFrame); + if( zLeft && strcmp(zLeft,"..end")==0 ){ + iEnd = mxFrame; + }else if( zLeft && zLeft[0]=='.' && zLeft[1]=='.' ){ + iEnd = strtol(&zLeft[2], 0, 0); + checkPageValidity(iEnd, mxFrame); + }else if( zLeft && zLeft[0]=='b' ){ + i64 ofst; + int nByte, hdrSize; + unsigned char *a; + if( iStart==1 ){ + hdrSize = 100; + ofst = hdrSize = 100; + nByte = pagesize-100; + }else{ + hdrSize = 0; + ofst = (i64)(iStart-1)*pagesize; + nByte = pagesize; + } + ofst = 32 + hdrSize + (i64)(iStart-1)*(pagesize+24) + 24; + a = getContent(ofst, nByte); + decode_btree_page(a, iStart, hdrSize, zLeft+1); + free(a); + continue; +#if !defined(_MSC_VER) + }else if( zLeft && strcmp(zLeft,"truncate")==0 ){ + /* Frame number followed by "truncate" truncates the WAL file + ** after that frame */ + off_t newSize = 32 + iStart*(pagesize+24); + truncate(argv[1], newSize); + continue; +#endif + }else{ + iEnd = iStart; + } + if( iStart<1 || iEnd<iStart || iEnd>mxFrame ){ + fprintf(stderr, + "Page argument should be LOWER?..UPPER?. Range 1 to %d\n", + mxFrame); + exit(1); + } + while( iStart<=iEnd ){ + print_frame(iStart); + iStart++; + } + } + } + close(fd); + return 0; +} diff --git a/tool/soak1.tcl b/tool/soak1.tcl new file mode 100644 index 0000000..846f905 --- /dev/null +++ b/tool/soak1.tcl @@ -0,0 +1,103 @@ +#!/usr/bin/tclsh +# +# Usage: +# +# tclsh soak1.tcl local-makefile.mk ?target? ?scenario? +# +# This generates many variations on local-makefile.mk (by modifing +# the OPT = lines) and runs them will fulltest, one by one. The +# constructed makefiles are named "soak1.mk". +# +# If ?target? is provided, that is the makefile target that is run. +# The default is "fulltest" +# +# If ?scenario? is provided, it is the name of a single scenario to +# be run. All other scenarios are skipped. +# +set localmake [lindex $argv 0] +set target [lindex $argv 1] +set scene [lindex $argv 2] +if {$target==""} {set target fulltest} +if {$scene==""} {set scene all} + +set in [open $localmake] +set maketxt [read $in] +close $in +regsub -all {\\\n} $maketxt {} maketxt +#set makefilename "soak1-[expr {int(rand()*1000000000)}].mk" +set makefilename "soak1.mk" + +# Generate a makefile +# +proc generate_makefile {pattern} { + global makefilename maketxt + set out [open $makefilename w] + set seen_opt 0 + foreach line [split $maketxt \n] { + if {[regexp {^ *#? *OPTS[ =+]} $line]} { + if {!$seen_opt} { + puts $out "OPTS += -DSQLITE_NO_SYNC=1" + foreach x $pattern { + puts $out "OPTS += -D$x" + } + set seen_opt 1 + } + } else { + puts $out $line + } + } + close $out +} + +# Run a test +# +proc scenario {id title pattern} { + global makefilename target scene + if {$scene!="all" && $scene!=$id && $scene!=$title} return + puts "**************** $title ***************" + generate_makefile $pattern + exec make -f $makefilename clean >@stdout 2>@stdout + exec make -f $makefilename $target >@stdout 2>@stdout +} + +############################################################################### +# Add new scenarios here +# +scenario 0 {Default} {} +scenario 1 {Debug} { + SQLITE_DEBUG=1 + SQLITE_MEMDEBUG=1 +} +scenario 2 {Everything} { + SQLITE_DEBUG=1 + SQLITE_MEMDEBUG=1 + SQLITE_ENABLE_MEMORY_MANAGEMENT=1 + SQLITE_ENABLE_COLUMN_METADATA=1 + SQLITE_ENABLE_LOAD_EXTENSION=1 HAVE_DLOPEN=1 + SQLITE_ENABLE_MEMORY_MANAGEMENT=1 +} +scenario 3 {Customer-1} { + SQLITE_DEBUG=1 SQLITE_MEMDEBUG=1 + SQLITE_THREADSAFE=1 SQLITE_OS_UNIX=1 + SQLITE_DISABLE_LFS=1 + SQLITE_DEFAULT_AUTOVACUUM=1 + SQLITE_DEFAULT_PAGE_SIZE=1024 + SQLITE_MAX_PAGE_SIZE=4096 + SQLITE_DEFAULT_CACHE_SIZE=64 + SQLITE_DEFAULT_TEMP_CACHE_SIZE=32 + SQLITE_TEMP_STORE=3 + SQLITE_OMIT_PROGRESS_CALLBACK=1 + SQLITE_OMIT_LOAD_EXTENSION=1 + SQLITE_OMIT_VIRTUALTABLE=1 + SQLITE_ENABLE_IOTRACE=1 +} +scenario 4 {Small-Cache} { + SQLITE_DEBUG=1 SQLITE_MEMDEBUG=1 + SQLITE_THREADSAFE=1 SQLITE_OS_UNIX=1 + SQLITE_DEFAULT_AUTOVACUUM=1 + SQLITE_DEFAULT_PAGE_SIZE=1024 + SQLITE_MAX_PAGE_SIZE=2048 + SQLITE_DEFAULT_CACHE_SIZE=13 + SQLITE_DEFAULT_TEMP_CACHE_SIZE=11 + SQLITE_TEMP_STORE=1 +} diff --git a/tool/spaceanal.tcl b/tool/spaceanal.tcl new file mode 100644 index 0000000..8fe72b9 --- /dev/null +++ b/tool/spaceanal.tcl @@ -0,0 +1,895 @@ +# Run this TCL script using an SQLite-enabled TCL interpreter to get a report +# on how much disk space is used by a particular data to actually store data +# versus how much space is unused. +# +# The dbstat virtual table is required. +# + +if {[catch { + +# Argument $tname is the name of a table within the database opened by +# database handle [db]. Return true if it is a WITHOUT ROWID table, or +# false otherwise. +# +proc is_without_rowid {tname} { + set t [string map {' ''} $tname] + db eval "PRAGMA index_list = '$t'" o { + if {$o(origin) == "pk"} { + set n $o(name) + if {0==[db one { SELECT count(*) FROM sqlite_schema WHERE name=$n }]} { + return 1 + } + } + } + return 0 +} + +# Read and run TCL commands from standard input. Used to implement +# the --tclsh option. +# +proc tclsh {} { + set line {} + while {![eof stdin]} { + if {$line!=""} { + puts -nonewline "> " + } else { + puts -nonewline "% " + } + flush stdout + append line [gets stdin] + if {[info complete $line]} { + if {[catch {uplevel #0 $line} result]} { + puts stderr "Error: $result" + } elseif {$result!=""} { + puts $result + } + set line {} + } else { + append line \n + } + } +} + + +# Get the name of the database to analyze +# +proc usage {} { + set argv0 [file rootname [file tail [info nameofexecutable]]] + puts stderr "Usage: $argv0 ?--pageinfo? ?--stats? database-filename" + puts stderr { +Analyze the SQLite3 database file specified by the "database-filename" +argument and output a report detailing size and storage efficiency +information for the database and its constituent tables and indexes. + +Options: + + --pageinfo Show how each page of the database-file is used + + --stats Output SQL text that creates a new database containing + statistics about the database that was analyzed + + --tclsh Run the built-in TCL interpreter interactively (for debugging) + + --version Show the version number of SQLite +} + exit 1 +} +set file_to_analyze {} +set flags(-pageinfo) 0 +set flags(-stats) 0 +set flags(-debug) 0 +append argv {} +foreach arg $argv { + if {[regexp {^-+pageinfo$} $arg]} { + set flags(-pageinfo) 1 + } elseif {[regexp {^-+stats$} $arg]} { + set flags(-stats) 1 + } elseif {[regexp {^-+debug$} $arg]} { + set flags(-debug) 1 + } elseif {[regexp {^-+tclsh$} $arg]} { + tclsh + exit 0 + } elseif {[regexp {^-+version$} $arg]} { + sqlite3 mem :memory: + puts [mem one {SELECT sqlite_version()||' '||sqlite_source_id()}] + mem close + exit 0 + } elseif {[regexp {^-} $arg]} { + puts stderr "Unknown option: $arg" + usage + } elseif {$file_to_analyze!=""} { + usage + } else { + set file_to_analyze $arg + } +} +if {$file_to_analyze==""} usage +set root_filename $file_to_analyze +regexp {^file:(//)?([^?]*)} $file_to_analyze all x1 root_filename +if {![file exists $root_filename]} { + puts stderr "No such file: $root_filename" + exit 1 +} +if {![file readable $root_filename]} { + puts stderr "File is not readable: $root_filename" + exit 1 +} +set true_file_size [file size $root_filename] +if {$true_file_size<512} { + puts stderr "Empty or malformed database: $root_filename" + exit 1 +} + +# Compute the total file size assuming test_multiplexor is being used. +# Assume that SQLITE_ENABLE_8_3_NAMES might be enabled +# +set extension [file extension $root_filename] +set pattern $root_filename +append pattern {[0-3][0-9][0-9]} +foreach f [glob -nocomplain $pattern] { + incr true_file_size [file size $f] + set extension {} +} +if {[string length $extension]>=2 && [string length $extension]<=4} { + set pattern [file rootname $root_filename] + append pattern {.[0-3][0-9][0-9]} + foreach f [glob -nocomplain $pattern] { + incr true_file_size [file size $f] + } +} + +# Open the database +# +if {[catch {sqlite3 db $file_to_analyze -uri 1} msg]} { + puts stderr "error trying to open $file_to_analyze: $msg" + exit 1 +} +if {$flags(-debug)} { + proc dbtrace {txt} {puts $txt; flush stdout;} + db trace ::dbtrace +} + +# Make sure all required compile-time options are available +# +if {![db exists {SELECT 1 FROM pragma_compile_options + WHERE compile_options='ENABLE_DBSTAT_VTAB'}]} { + puts "The SQLite database engine linked with this application\ + lacks required capabilities. Recompile using the\ + -DSQLITE_ENABLE_DBSTAT_VTAB compile-time option to fix\ + this problem." + exit 1 +} + +db eval {SELECT count(*) FROM sqlite_schema} +set pageSize [expr {wide([db one {PRAGMA page_size}])}] + +if {$flags(-pageinfo)} { + db eval {CREATE VIRTUAL TABLE temp.stat USING dbstat} + db eval {SELECT name, path, pageno FROM temp.stat ORDER BY pageno} { + puts "$pageno $name $path" + } + exit 0 +} +if {$flags(-stats)} { + db eval {CREATE VIRTUAL TABLE temp.stat USING dbstat} + puts "BEGIN;" + puts "CREATE TABLE stats(" + puts " name STRING, /* Name of table or index */" + puts " path INTEGER, /* Path to page from root */" + puts " pageno INTEGER, /* Page number */" + puts " pagetype STRING, /* 'internal', 'leaf' or 'overflow' */" + puts " ncell INTEGER, /* Cells on page (0 for overflow) */" + puts " payload INTEGER, /* Bytes of payload on this page */" + puts " unused INTEGER, /* Bytes of unused space on this page */" + puts " mx_payload INTEGER, /* Largest payload size of all cells */" + puts " pgoffset INTEGER, /* Offset of page in file */" + puts " pgsize INTEGER /* Size of the page */" + puts ");" + db eval {SELECT quote(name) || ',' || + quote(path) || ',' || + quote(pageno) || ',' || + quote(pagetype) || ',' || + quote(ncell) || ',' || + quote(payload) || ',' || + quote(unused) || ',' || + quote(mx_payload) || ',' || + quote(pgoffset) || ',' || + quote(pgsize) AS x FROM stat} { + puts "INSERT INTO stats VALUES($x);" + } + puts "COMMIT;" + exit 0 +} + + +# In-memory database for collecting statistics. This script loops through +# the tables and indices in the database being analyzed, adding a row for each +# to an in-memory database (for which the schema is shown below). It then +# queries the in-memory db to produce the space-analysis report. +# +sqlite3 mem :memory: +if {$flags(-debug)} { + proc dbtrace {txt} {puts $txt; flush stdout;} + mem trace ::dbtrace +} +set tabledef {CREATE TABLE space_used( + name clob, -- Name of a table or index in the database file + tblname clob, -- Name of associated table + is_index boolean, -- TRUE if it is an index, false for a table + is_without_rowid boolean, -- TRUE if WITHOUT ROWID table + nentry int, -- Number of entries in the BTree + leaf_entries int, -- Number of leaf entries + depth int, -- Depth of the b-tree + payload int, -- Total amount of data stored in this table or index + ovfl_payload int, -- Total amount of data stored on overflow pages + ovfl_cnt int, -- Number of entries that use overflow + mx_payload int, -- Maximum payload size + int_pages int, -- Number of interior pages used + leaf_pages int, -- Number of leaf pages used + ovfl_pages int, -- Number of overflow pages used + int_unused int, -- Number of unused bytes on interior pages + leaf_unused int, -- Number of unused bytes on primary pages + ovfl_unused int, -- Number of unused bytes on overflow pages + gap_cnt int, -- Number of gaps in the page layout + compressed_size int -- Total bytes stored on disk +);} +mem eval $tabledef + +# Create a temporary "dbstat" virtual table. +# +db eval {CREATE VIRTUAL TABLE temp.stat USING dbstat} +db eval {CREATE TEMP TABLE dbstat AS SELECT * FROM temp.stat + ORDER BY name, path} +db eval {DROP TABLE temp.stat} + +set isCompressed 0 +set compressOverhead 0 +set depth 0 +set sql { SELECT name, tbl_name FROM sqlite_schema WHERE rootpage>0 } +foreach {name tblname} [concat sqlite_schema sqlite_schema [db eval $sql]] { + + set is_index [expr {$name!=$tblname}] + set is_without_rowid [is_without_rowid $name] + db eval { + SELECT + sum(ncell) AS nentry, + sum((pagetype=='leaf')*ncell) AS leaf_entries, + sum(payload) AS payload, + sum((pagetype=='overflow') * payload) AS ovfl_payload, + sum(path LIKE '%+000000') AS ovfl_cnt, + max(mx_payload) AS mx_payload, + sum(pagetype=='internal') AS int_pages, + sum(pagetype=='leaf') AS leaf_pages, + sum(pagetype=='overflow') AS ovfl_pages, + sum((pagetype=='internal') * unused) AS int_unused, + sum((pagetype=='leaf') * unused) AS leaf_unused, + sum((pagetype=='overflow') * unused) AS ovfl_unused, + sum(pgsize) AS compressed_size, + max((length(CASE WHEN path LIKE '%+%' THEN '' ELSE path END)+3)/4) + AS depth + FROM temp.dbstat WHERE name = $name + } break + + set total_pages [expr {$leaf_pages+$int_pages+$ovfl_pages}] + set storage [expr {$total_pages*$pageSize}] + if {!$isCompressed && $storage>$compressed_size} { + set isCompressed 1 + set compressOverhead 14 + } + + # Column 'gap_cnt' is set to the number of non-contiguous entries in the + # list of pages visited if the b-tree structure is traversed in a top-down + # fashion (each node visited before its child-tree is passed). Any overflow + # chains present are traversed from start to finish before any child-tree + # is. + # + set gap_cnt 0 + set prev 0 + db eval { + SELECT pageno, pagetype FROM temp.dbstat + WHERE name=$name + ORDER BY pageno + } { + if {$prev>0 && $pagetype=="leaf" && $pageno!=$prev+1} { + incr gap_cnt + } + set prev $pageno + } + mem eval { + INSERT INTO space_used VALUES( + $name, + $tblname, + $is_index, + $is_without_rowid, + $nentry, + $leaf_entries, + $depth, + $payload, + $ovfl_payload, + $ovfl_cnt, + $mx_payload, + $int_pages, + $leaf_pages, + $ovfl_pages, + $int_unused, + $leaf_unused, + $ovfl_unused, + $gap_cnt, + $compressed_size + ); + } +} + +proc integerify {real} { + if {[string is double -strict $real]} { + return [expr {wide($real)}] + } else { + return 0 + } +} +mem function int integerify + +# Quote a string for use in an SQL query. Examples: +# +# [quote {hello world}] == {'hello world'} +# [quote {hello world's}] == {'hello world''s'} +# +proc quote {txt} { + return [string map {' ''} $txt] +} + +# Output a title line +# +proc titleline {title} { + if {$title==""} { + puts [string repeat * 79] + } else { + set len [string length $title] + set stars [string repeat * [expr 79-$len-5]] + puts "*** $title $stars" + } +} + +# Generate a single line of output in the statistics section of the +# report. +# +proc statline {title value {extra {}}} { + set len [string length $title] + set dots [string repeat . [expr 50-$len]] + set len [string length $value] + set sp2 [string range { } $len end] + if {$extra ne ""} { + set extra " $extra" + } + puts "$title$dots $value$sp2$extra" +} + +# Generate a formatted percentage value for $num/$denom +# +proc percent {num denom {of {}}} { + if {$denom==0.0} {return ""} + set v [expr {$num*100.0/$denom}] + set of {} + if {$v==100.0 || $v<0.001 || ($v>1.0 && $v<99.0)} { + return [format {%5.1f%% %s} $v $of] + } elseif {$v<0.1 || $v>99.9} { + return [format {%7.3f%% %s} $v $of] + } else { + return [format {%6.2f%% %s} $v $of] + } +} + +proc divide {num denom} { + if {$denom==0} {return 0.0} + return [format %.2f [expr double($num)/double($denom)]] +} + +# Generate a subreport that covers some subset of the database. +# the $where clause determines which subset to analyze. +# +proc subreport {title where showFrag} { + global pageSize file_pgcnt compressOverhead + + # Query the in-memory database for the sum of various statistics + # for the subset of tables/indices identified by the WHERE clause in + # $where. Note that even if the WHERE clause matches no rows, the + # following query returns exactly one row (because it is an aggregate). + # + # The results of the query are stored directly by SQLite into local + # variables (i.e. $nentry, $payload etc.). + # + mem eval " + SELECT + int(sum( + CASE WHEN (is_without_rowid OR is_index) THEN nentry + ELSE leaf_entries + END + )) AS nentry, + int(sum(payload)) AS payload, + int(sum(ovfl_payload)) AS ovfl_payload, + max(mx_payload) AS mx_payload, + int(sum(ovfl_cnt)) as ovfl_cnt, + int(sum(leaf_pages)) AS leaf_pages, + int(sum(int_pages)) AS int_pages, + int(sum(ovfl_pages)) AS ovfl_pages, + int(sum(leaf_unused)) AS leaf_unused, + int(sum(int_unused)) AS int_unused, + int(sum(ovfl_unused)) AS ovfl_unused, + int(sum(gap_cnt)) AS gap_cnt, + int(sum(compressed_size)) AS compressed_size, + int(max(depth)) AS depth, + count(*) AS cnt + FROM space_used WHERE $where" {} {} + + # Output the sub-report title, nicely decorated with * characters. + # + puts "" + titleline $title + puts "" + + # Calculate statistics and store the results in TCL variables, as follows: + # + # total_pages: Database pages consumed. + # total_pages_percent: Pages consumed as a percentage of the file. + # storage: Bytes consumed. + # payload_percent: Payload bytes used as a percentage of $storage. + # total_unused: Unused bytes on pages. + # avg_payload: Average payload per btree entry. + # avg_fanout: Average fanout for internal pages. + # avg_unused: Average unused bytes per btree entry. + # avg_meta: Average metadata overhead per entry. + # ovfl_cnt_percent: Percentage of btree entries that use overflow pages. + # + set total_pages [expr {$leaf_pages+$int_pages+$ovfl_pages}] + set total_pages_percent [percent $total_pages $file_pgcnt] + set storage [expr {$total_pages*$pageSize}] + set payload_percent [percent $payload $storage {of storage consumed}] + set total_unused [expr {$ovfl_unused+$int_unused+$leaf_unused}] + set avg_payload [divide $payload $nentry] + set avg_unused [divide $total_unused $nentry] + set total_meta [expr {$storage - $payload - $total_unused}] + set total_meta [expr {$total_meta + 4*($ovfl_pages - $ovfl_cnt)}] + set meta_percent [percent $total_meta $storage {of metadata}] + set avg_meta [divide $total_meta $nentry] + if {$int_pages>0} { + # TODO: Is this formula correct? + set nTab [mem eval " + SELECT count(*) FROM ( + SELECT DISTINCT tblname FROM space_used WHERE $where AND is_index=0 + ) + "] + set avg_fanout [mem eval " + SELECT (sum(leaf_pages+int_pages)-$nTab)/sum(int_pages) FROM space_used + WHERE $where + "] + set avg_fanout [format %.2f $avg_fanout] + } + set ovfl_cnt_percent [percent $ovfl_cnt $nentry {of all entries}] + + # Print out the sub-report statistics. + # + statline {Percentage of total database} $total_pages_percent + statline {Number of entries} $nentry + statline {Bytes of storage consumed} $storage + if {$compressed_size!=$storage} { + set compressed_size [expr {$compressed_size+$compressOverhead*$total_pages}] + set pct [expr {$compressed_size*100.0/$storage}] + set pct [format {%5.1f%%} $pct] + statline {Bytes used after compression} $compressed_size $pct + } + statline {Bytes of payload} $payload $payload_percent + statline {Bytes of metadata} $total_meta $meta_percent + if {$cnt==1} {statline {B-tree depth} $depth} + statline {Average payload per entry} $avg_payload + statline {Average unused bytes per entry} $avg_unused + statline {Average metadata per entry} $avg_meta + if {[info exists avg_fanout]} { + statline {Average fanout} $avg_fanout + } + if {$showFrag && $total_pages>1} { + set fragmentation [percent $gap_cnt [expr {$total_pages-1}]] + statline {Non-sequential pages} $gap_cnt $fragmentation + } + statline {Maximum payload per entry} $mx_payload + statline {Entries that use overflow} $ovfl_cnt $ovfl_cnt_percent + if {$int_pages>0} { + statline {Index pages used} $int_pages + } + statline {Primary pages used} $leaf_pages + statline {Overflow pages used} $ovfl_pages + statline {Total pages used} $total_pages + if {$int_unused>0} { + set int_unused_percent [ + percent $int_unused [expr {$int_pages*$pageSize}] {of index space}] + statline "Unused bytes on index pages" $int_unused $int_unused_percent + } + statline "Unused bytes on primary pages" $leaf_unused [ + percent $leaf_unused [expr {$leaf_pages*$pageSize}] {of primary space}] + statline "Unused bytes on overflow pages" $ovfl_unused [ + percent $ovfl_unused [expr {$ovfl_pages*$pageSize}] {of overflow space}] + statline "Unused bytes on all pages" $total_unused [ + percent $total_unused $storage {of all space}] + return 1 +} + +# Calculate the overhead in pages caused by auto-vacuum. +# +# This procedure calculates and returns the number of pages used by the +# auto-vacuum 'pointer-map'. If the database does not support auto-vacuum, +# then 0 is returned. The two arguments are the size of the database file in +# pages and the page size used by the database (in bytes). +proc autovacuum_overhead {filePages pageSize} { + + # Set $autovacuum to non-zero for databases that support auto-vacuum. + set autovacuum [db one {PRAGMA auto_vacuum}] + + # If the database is not an auto-vacuum database or the file consists + # of one page only then there is no overhead for auto-vacuum. Return zero. + if {0==$autovacuum || $filePages==1} { + return 0 + } + + # The number of entries on each pointer map page. The layout of the + # database file is one pointer-map page, followed by $ptrsPerPage other + # pages, followed by a pointer-map page etc. The first pointer-map page + # is the second page of the file overall. + set ptrsPerPage [expr double($pageSize/5)] + + # Return the number of pointer map pages in the database. + return [expr wide(ceil( ($filePages-1.0)/($ptrsPerPage+1.0) ))] +} + + +# Calculate the summary statistics for the database and store the results +# in TCL variables. They are output below. Variables are as follows: +# +# pageSize: Size of each page in bytes. +# file_bytes: File size in bytes. +# file_pgcnt: Number of pages in the file. +# file_pgcnt2: Number of pages in the file (calculated). +# av_pgcnt: Pages consumed by the auto-vacuum pointer-map. +# av_percent: Percentage of the file consumed by auto-vacuum pointer-map. +# inuse_pgcnt: Data pages in the file. +# inuse_percent: Percentage of pages used to store data. +# free_pgcnt: Free pages calculated as (<total pages> - <in-use pages>) +# free_pgcnt2: Free pages in the file according to the file header. +# free_percent: Percentage of file consumed by free pages (calculated). +# free_percent2: Percentage of file consumed by free pages (header). +# ntable: Number of tables in the db. +# nindex: Number of indices in the db. +# nautoindex: Number of indices created automatically. +# nmanindex: Number of indices created manually. +# user_payload: Number of bytes of payload in table btrees +# (not including sqlite_schema) +# user_percent: $user_payload as a percentage of total file size. + +### The following, setting $file_bytes based on the actual size of the file +### on disk, causes this tool to choke on zipvfs databases. So set it based +### on the return of [PRAGMA page_count] instead. +if 0 { + set file_bytes [file size $file_to_analyze] + set file_pgcnt [expr {$file_bytes/$pageSize}] +} +set file_pgcnt [db one {PRAGMA page_count}] +set file_bytes [expr {$file_pgcnt * $pageSize}] + +set av_pgcnt [autovacuum_overhead $file_pgcnt $pageSize] +set av_percent [percent $av_pgcnt $file_pgcnt] + +set sql {SELECT sum(leaf_pages+int_pages+ovfl_pages) FROM space_used} +set inuse_pgcnt [expr wide([mem eval $sql])] +set inuse_percent [percent $inuse_pgcnt $file_pgcnt] + +set free_pgcnt [expr {$file_pgcnt-$inuse_pgcnt-$av_pgcnt}] +if {$file_bytes>1073741824 && $free_pgcnt>0} {incr free_pgcnt -1} +set free_percent [percent $free_pgcnt $file_pgcnt] +set free_pgcnt2 [db one {PRAGMA freelist_count}] +set free_percent2 [percent $free_pgcnt2 $file_pgcnt] + +set file_pgcnt2 [expr {$inuse_pgcnt+$free_pgcnt2+$av_pgcnt}] + +# Account for the lockbyte page +if {$file_pgcnt2*$pageSize>1073742335} {incr file_pgcnt2} + +set ntable [db eval {SELECT count(*)+1 FROM sqlite_schema WHERE type='table'}] +set nindex [db eval {SELECT count(*) FROM sqlite_schema WHERE type='index'}] +set sql {SELECT count(*) FROM sqlite_schema WHERE name LIKE 'sqlite_autoindex%'} +set nautoindex [db eval $sql] +set nmanindex [expr {$nindex-$nautoindex}] + +# set total_payload [mem eval "SELECT sum(payload) FROM space_used"] +set user_payload [mem one {SELECT int(sum(payload)) FROM space_used + WHERE NOT is_index AND name NOT LIKE 'sqlite_schema'}] +set user_percent [percent $user_payload $file_bytes] + +# Output the summary statistics calculated above. +# +puts "/** Disk-Space Utilization Report For $root_filename" +puts "" +statline {Page size in bytes} $pageSize +statline {Pages in the whole file (measured)} $file_pgcnt +statline {Pages in the whole file (calculated)} $file_pgcnt2 +statline {Pages that store data} $inuse_pgcnt $inuse_percent +statline {Pages on the freelist (per header)} $free_pgcnt2 $free_percent2 +statline {Pages on the freelist (calculated)} $free_pgcnt $free_percent +statline {Pages of auto-vacuum overhead} $av_pgcnt $av_percent +statline {Number of tables in the database} $ntable +statline {Number of indices} $nindex +statline {Number of defined indices} $nmanindex +statline {Number of implied indices} $nautoindex +if {$isCompressed} { + statline {Size of uncompressed content in bytes} $file_bytes + set efficiency [percent $true_file_size $file_bytes] + statline {Size of compressed file on disk} $true_file_size $efficiency +} else { + statline {Size of the file in bytes} $file_bytes +} +statline {Bytes of user payload stored} $user_payload $user_percent + +# Output table rankings +# +puts "" +titleline "Page counts for all tables with their indices" +puts "" +mem eval {SELECT tblname, count(*) AS cnt, + int(sum(int_pages+leaf_pages+ovfl_pages)) AS size + FROM space_used GROUP BY tblname ORDER BY size+0 DESC, tblname} {} { + statline [string toupper $tblname] $size [percent $size $file_pgcnt] +} +puts "" +titleline "Page counts for all tables and indices separately" +puts "" +mem eval { + SELECT + upper(name) AS nm, + int(int_pages+leaf_pages+ovfl_pages) AS size + FROM space_used + ORDER BY size+0 DESC, name} {} { + statline $nm $size [percent $size $file_pgcnt] +} +if {$isCompressed} { + puts "" + titleline "Bytes of disk space used after compression" + puts "" + set csum 0 + mem eval {SELECT tblname, + int(sum(compressed_size)) + + $compressOverhead*sum(int_pages+leaf_pages+ovfl_pages) + AS csize + FROM space_used GROUP BY tblname ORDER BY csize+0 DESC, tblname} {} { + incr csum $csize + statline [string toupper $tblname] $csize [percent $csize $true_file_size] + } + set overhead [expr {$true_file_size - $csum}] + if {$overhead>0} { + statline {Header and free space} $overhead [percent $overhead $true_file_size] + } +} + +# Output subreports +# +if {$nindex>0} { + subreport {All tables and indices} 1 0 +} +subreport {All tables} {NOT is_index} 0 +if {$nindex>0} { + subreport {All indices} {is_index} 0 +} +foreach tbl [mem eval {SELECT DISTINCT tblname name FROM space_used + ORDER BY name}] { + set qn [quote $tbl] + set name [string toupper $tbl] + set n [mem eval {SELECT count(*) FROM space_used WHERE tblname=$tbl}] + if {$n>1} { + set idxlist [mem eval "SELECT name FROM space_used + WHERE tblname='$qn' AND is_index + ORDER BY 1"] + subreport "Table $name and all its indices" "tblname='$qn'" 0 + subreport "Table $name w/o any indices" "name='$qn'" 1 + if {[llength $idxlist]>1} { + subreport "Indices of table $name" "tblname='$qn' AND is_index" 0 + } + foreach idx $idxlist { + set qidx [quote $idx] + subreport "Index [string toupper $idx] of table $name" "name='$qidx'" 1 + } + } else { + subreport "Table $name" "name='$qn'" 1 + } +} + +# Output instructions on what the numbers above mean. +# +puts "" +titleline Definitions +puts { +Page size in bytes + + The number of bytes in a single page of the database file. + Usually 1024. + +Number of pages in the whole file +} +puts " The number of $pageSize-byte pages that go into forming the complete + database" +puts { +Pages that store data + + The number of pages that store data, either as primary B*Tree pages or + as overflow pages. The number at the right is the data pages divided by + the total number of pages in the file. + +Pages on the freelist + + The number of pages that are not currently in use but are reserved for + future use. The percentage at the right is the number of freelist pages + divided by the total number of pages in the file. + +Pages of auto-vacuum overhead + + The number of pages that store data used by the database to facilitate + auto-vacuum. This is zero for databases that do not support auto-vacuum. + +Number of tables in the database + + The number of tables in the database, including the SQLITE_SCHEMA table + used to store schema information. + +Number of indices + + The total number of indices in the database. + +Number of defined indices + + The number of indices created using an explicit CREATE INDEX statement. + +Number of implied indices + + The number of indices used to implement PRIMARY KEY or UNIQUE constraints + on tables. + +Size of the file in bytes + + The total amount of disk space used by the entire database files. + +Bytes of user payload stored + + The total number of bytes of user payload stored in the database. The + schema information in the SQLITE_SCHEMA table is not counted when + computing this number. The percentage at the right shows the payload + divided by the total file size. + +Percentage of total database + + The amount of the complete database file that is devoted to storing + information described by this category. + +Number of entries + + The total number of B-Tree key/value pairs stored under this category. + +Bytes of storage consumed + + The total amount of disk space required to store all B-Tree entries + under this category. The is the total number of pages used times + the pages size. + +Bytes of payload + + The amount of payload stored under this category. Payload is the data + part of table entries and the key part of index entries. The percentage + at the right is the bytes of payload divided by the bytes of storage + consumed. + +Bytes of metadata + + The amount of formatting and structural information stored in the + table or index. Metadata includes the btree page header, the cell pointer + array, the size field for each cell, the left child pointer or non-leaf + cells, the overflow pointers for overflow cells, and the rowid value for + rowid table cells. In other words, metadata is everything that is neither + unused space nor content. The record header in the payload is counted as + content, not metadata. + +Average payload per entry + + The average amount of payload on each entry. This is just the bytes of + payload divided by the number of entries. + +Average unused bytes per entry + + The average amount of free space remaining on all pages under this + category on a per-entry basis. This is the number of unused bytes on + all pages divided by the number of entries. + +Non-sequential pages + + The number of pages in the table or index that are out of sequence. + Many filesystems are optimized for sequential file access so a small + number of non-sequential pages might result in faster queries, + especially for larger database files that do not fit in the disk cache. + Note that after running VACUUM, the root page of each table or index is + at the beginning of the database file and all other pages are in a + separate part of the database file, resulting in a single non- + sequential page. + +Maximum payload per entry + + The largest payload size of any entry. + +Entries that use overflow + + The number of entries that user one or more overflow pages. + +Total pages used + + This is the number of pages used to hold all information in the current + category. This is the sum of index, primary, and overflow pages. + +Index pages used + + This is the number of pages in a table B-tree that hold only key (rowid) + information and no data. + +Primary pages used + + This is the number of B-tree pages that hold both key and data. + +Overflow pages used + + The total number of overflow pages used for this category. + +Unused bytes on index pages + + The total number of bytes of unused space on all index pages. The + percentage at the right is the number of unused bytes divided by the + total number of bytes on index pages. + +Unused bytes on primary pages + + The total number of bytes of unused space on all primary pages. The + percentage at the right is the number of unused bytes divided by the + total number of bytes on primary pages. + +Unused bytes on overflow pages + + The total number of bytes of unused space on all overflow pages. The + percentage at the right is the number of unused bytes divided by the + total number of bytes on overflow pages. + +Unused bytes on all pages + + The total number of bytes of unused space on all primary and overflow + pages. The percentage at the right is the number of unused bytes + divided by the total number of bytes. +} + +# Output a dump of the in-memory database. This can be used for more +# complex offline analysis. +# +titleline {} +puts "The entire text of this report can be sourced into any SQL database" +puts "engine for further analysis. All of the text above is an SQL comment." +puts "The data used to generate this report follows:" +puts "*/" +puts "BEGIN;" +puts $tabledef +unset -nocomplain x +mem eval {SELECT * FROM space_used} x { + puts -nonewline "INSERT INTO space_used VALUES" + set sep ( + foreach col $x(*) { + set v $x($col) + if {$v=="" || ![string is double $v]} {set v '[quote $v]'} + puts -nonewline $sep$v + set sep , + } + puts ");" +} +puts "COMMIT;" + +} err]} { + puts "ERROR: $err" + puts $errorInfo + exit 1 +} diff --git a/tool/speed-check.sh b/tool/speed-check.sh new file mode 100644 index 0000000..4cc2579 --- /dev/null +++ b/tool/speed-check.sh @@ -0,0 +1,219 @@ +#!/bin/bash +# +# This is a template for a script used for day-to-day size and +# performance monitoring of SQLite. Typical usage: +# +# sh speed-check.sh trunk # Baseline measurement of trunk +# sh speed-check.sh x1 # Measure some experimental change +# fossil xdiff --tk cout-trunk.txt cout-x1.txt # View chanages +# +# There are multiple output files, all with a base name given by +# the first argument: +# +# summary-$BASE.txt # Copy of standard output +# cout-$BASE.txt # cachegrind output +# explain-$BASE.txt # EXPLAIN listings (only with --explain) +# +if test "$1" = "" +then + echo "Usage: $0 OUTPUTFILE [OPTIONS]" + exit +fi +NAME=$1 +shift +#CC_OPTS="-DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_MEMSYS5" +CC_OPTS="-DSQLITE_ENABLE_MEMSYS5" +CC=gcc +SPEEDTEST_OPTS="--shrink-memory --reprepare --stats --heap 10000000 64" +SIZE=5 +LEAN_OPTS="-DSQLITE_THREADSAFE=0" +LEAN_OPTS="$LEAN_OPTS -DSQLITE_DEFAULT_MEMSTATUS=0" +LEAN_OPTS="$LEAN_OPTS -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1" +LEAN_OPTS="$LEAN_OPTS -DSQLITE_LIKE_DOESNT_MATCH_BLOBS" +LEAN_OPTS="$LEAN_OPTS -DSQLITE_MAX_EXPR_DEPTH=0" +LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_DECLTYPE" +LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_DEPRECATED" +LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_PROGRESS_CALLBACK" +LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_SHARED_CACHE" +LEAN_OPTS="$LEAN_OPTS -DSQLITE_USE_ALLOCA" +BASELINE="trunk" +doExplain=0 +doCachegrind=1 +doVdbeProfile=0 +doWal=1 +doDiff=1 +while test "$1" != ""; do + case $1 in + --nodiff) + doDiff=0 + ;; + --reprepare) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" + ;; + --autovacuum) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" + ;; + --utf16be) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" + ;; + --stats) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" + ;; + --without-rowid) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" + ;; + --strict) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" + ;; + --nomemstat) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" + ;; + --multithread) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" + ;; + --singlethread) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" + ;; + --serialized) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" + ;; + --temp) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --temp 6" + ;; + --legacy) + doWal=0 + CC_OPTS="$CC_OPTS -DSPEEDTEST_OMIT_HASH" + ;; + --verify) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --verify" + ;; + --wal) + doWal=1 + ;; + --size) + shift; SIZE=$1 + ;; + --cachesize) + shift; SPEEDTEST_OPTS="$SPEEDTEST_OPTS --cachesize $1" + ;; + --stmtcache) + shift; SPEEDTEST_OPTS="$SPEEDTEST_OPTS --stmtcache $1" + ;; + --nolongdouble) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --nolongdouble" + ;; + --checkpoint) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --checkpoint" + ;; + --explain) + doExplain=1 + ;; + --vdbeprofile) + rm -f vdbe_profile.out + CC_OPTS="$CC_OPTS -DVDBE_PROFILE" + doCachegrind=0 + doVdbeProfile=1 + ;; + --lean) + CC_OPTS="$CC_OPTS $LEAN_OPTS" + ;; + --clang) + CC=clang + ;; + --icc) + CC=/home/drh/intel/bin/icc + ;; + --gcc7) + CC=gcc-7 + ;; + --heap) + CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_MEMSYS5" + shift; + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --heap $1 64" + ;; + --lookaside) + shift; + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --lookaside $1 $2" + shift; + ;; + --repeat) + CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_RCACHE" + shift; + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --repeat $1" + ;; + --mmap) + shift; + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --mmap $1" + ;; + --rtree) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset rtree" + CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_RTREE" + ;; + --persist) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --persist" + ;; + --orm) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset orm" + ;; + --cte) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset cte" + ;; + --fp) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset fp" + ;; + --stmtscanstatus) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --stmtscanstatus" + ;; + -*) + CC_OPTS="$CC_OPTS $1" + ;; + *) + BASELINE=$1 + ;; + esac + shift +done +if test $doWal -eq 1; then + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --journal wal" +fi +SPEEDTEST_OPTS="$SPEEDTEST_OPTS --size $SIZE" +echo "NAME = $NAME" | tee summary-$NAME.txt +echo "SPEEDTEST_OPTS = $SPEEDTEST_OPTS" | tee -a summary-$NAME.txt +echo "CC_OPTS = $CC_OPTS" | tee -a summary-$NAME.txt +rm -f cachegrind.out.* speedtest1 speedtest1.db sqlite3.o +if test $doVdbeProfile -eq 1; then + rm -f vdbe_profile.out +fi +$CC -g -Os -Wall -I. $CC_OPTS -c sqlite3.c +size sqlite3.o | tee -a summary-$NAME.txt +if test $doExplain -eq 1; then + $CC -g -Os -Wall -I. $CC_OPTS \ + -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ + ./shell.c ./sqlite3.c -o sqlite3 -ldl -lpthread +fi +SRC=./speedtest1.c +$CC -g -Os -Wall -I. $CC_OPTS $SRC ./sqlite3.o -o speedtest1 -ldl -lpthread +ls -l speedtest1 | tee -a summary-$NAME.txt +if test $doCachegrind -eq 1; then + valgrind --tool=cachegrind ./speedtest1 speedtest1.db \ + $SPEEDTEST_OPTS 2>&1 | tee -a summary-$NAME.txt +else + ./speedtest1 speedtest1.db $SPEEDTEST_OPTS 2>&1 | tee -a summary-$NAME.txt +fi +size sqlite3.o | tee -a summary-$NAME.txt +wc sqlite3.c +if test $doCachegrind -eq 1; then + cg_anno.tcl cachegrind.out.* >cout-$NAME.txt + echo '*****************************************************' >>cout-$NAME.txt + sed 's/^[0-9=-]\{9\}/==00000==/' summary-$NAME.txt >>cout-$NAME.txt +fi +if test $doExplain -eq 1; then + ./speedtest1 --explain $SPEEDTEST_OPTS | ./sqlite3 >explain-$NAME.txt +fi +if test $doVdbeProfile -eq 1; then + tclsh ../sqlite/tool/vdbe_profile.tcl >vdbeprofile-$NAME.txt + open vdbeprofile-$NAME.txt +fi +if test "$NAME" != "$BASELINE" -a $doVdbeProfile -ne 1 -a $doDiff -ne 0; then + fossil test-diff --tk -c 20 cout-$BASELINE.txt cout-$NAME.txt +fi diff --git a/tool/speedtest.tcl b/tool/speedtest.tcl new file mode 100644 index 0000000..ef39dc5 --- /dev/null +++ b/tool/speedtest.tcl @@ -0,0 +1,275 @@ +#!/usr/bin/tclsh +# +# Run this script using TCLSH to do a speed comparison between +# various versions of SQLite and PostgreSQL and MySQL +# + +# Run a test +# +set cnt 1 +proc runtest {title} { + global cnt + set sqlfile test$cnt.sql + puts "<h2>Test $cnt: $title</h2>" + incr cnt + set fd [open $sqlfile r] + set sql [string trim [read $fd [file size $sqlfile]]] + close $fd + set sx [split $sql \n] + set n [llength $sx] + if {$n>8} { + set sql {} + for {set i 0} {$i<3} {incr i} {append sql [lindex $sx $i]<br>\n} + append sql "<i>... [expr {$n-6}] lines omitted</i><br>\n" + for {set i [expr {$n-3}]} {$i<$n} {incr i} { + append sql [lindex $sx $i]<br>\n + } + } else { + regsub -all \n [string trim $sql] <br> sql + } + puts "<blockquote>" + puts "$sql" + puts "</blockquote><table border=0 cellpadding=0 cellspacing=0>" + set format {<tr><td>%s</td><td align="right"> %.3f</td></tr>} + set delay 1000 +# exec sync; after $delay; +# set t [time "exec psql drh <$sqlfile" 1] +# set t [expr {[lindex $t 0]/1000000.0}] +# puts [format $format PostgreSQL: $t] + exec sync; after $delay; + set t [time "exec mysql -f drh <$sqlfile" 1] + set t [expr {[lindex $t 0]/1000000.0}] + puts [format $format MySQL: $t] +# set t [time "exec ./sqlite232 s232.db <$sqlfile" 1] +# set t [expr {[lindex $t 0]/1000000.0}] +# puts [format $format {SQLite 2.3.2:} $t] +# set t [time "exec ./sqlite-100 s100.db <$sqlfile" 1] +# set t [expr {[lindex $t 0]/1000000.0}] +# puts [format $format {SQLite 2.4 (cache=100):} $t] + exec sync; after $delay; + set t [time "exec ./sqlite248 s2k.db <$sqlfile" 1] + set t [expr {[lindex $t 0]/1000000.0}] + puts [format $format {SQLite 2.4.8:} $t] + exec sync; after $delay; + set t [time "exec ./sqlite248 sns.db <$sqlfile" 1] + set t [expr {[lindex $t 0]/1000000.0}] + puts [format $format {SQLite 2.4.8 (nosync):} $t] + exec sync; after $delay; + set t [time "exec ./sqlite2412 s2kb.db <$sqlfile" 1] + set t [expr {[lindex $t 0]/1000000.0}] + puts [format $format {SQLite 2.4.12:} $t] + exec sync; after $delay; + set t [time "exec ./sqlite2412 snsb.db <$sqlfile" 1] + set t [expr {[lindex $t 0]/1000000.0}] + puts [format $format {SQLite 2.4.12 (nosync):} $t] +# set t [time "exec ./sqlite-t1 st1.db <$sqlfile" 1] +# set t [expr {[lindex $t 0]/1000000.0}] +# puts [format $format {SQLite 2.4 (test):} $t] + puts "</table>" +} + +# Initialize the environment +# +expr srand(1) +catch {exec /bin/sh -c {rm -f s*.db}} +set fd [open clear.sql w] +puts $fd { + drop table t1; + drop table t2; +} +close $fd +catch {exec psql drh <clear.sql} +catch {exec mysql drh <clear.sql} +set fd [open 2kinit.sql w] +puts $fd { + PRAGMA default_cache_size=2000; + PRAGMA default_synchronous=on; +} +close $fd +exec ./sqlite248 s2k.db <2kinit.sql +exec ./sqlite2412 s2kb.db <2kinit.sql +set fd [open nosync-init.sql w] +puts $fd { + PRAGMA default_cache_size=2000; + PRAGMA default_synchronous=off; +} +close $fd +exec ./sqlite248 sns.db <nosync-init.sql +exec ./sqlite2412 snsb.db <nosync-init.sql +set ones {zero one two three four five six seven eight nine + ten eleven twelve thirteen fourteen fifteen sixteen seventeen + eighteen nineteen} +set tens {{} ten twenty thirty forty fifty sixty seventy eighty ninety} +proc number_name {n} { + if {$n>=1000} { + set txt "[number_name [expr {$n/1000}]] thousand" + set n [expr {$n%1000}] + } else { + set txt {} + } + if {$n>=100} { + append txt " [lindex $::ones [expr {$n/100}]] hundred" + set n [expr {$n%100}] + } + if {$n>=20} { + append txt " [lindex $::tens [expr {$n/10}]]" + set n [expr {$n%10}] + } + if {$n>0} { + append txt " [lindex $::ones $n]" + } + set txt [string trim $txt] + if {$txt==""} {set txt zero} + return $txt +} + + + +set fd [open test$cnt.sql w] +puts $fd "CREATE TABLE t1(a INTEGER, b INTEGER, c VARCHAR(100));" +for {set i 1} {$i<=1000} {incr i} { + set r [expr {int(rand()*100000)}] + puts $fd "INSERT INTO t1 VALUES($i,$r,'[number_name $r]');" +} +close $fd +runtest {1000 INSERTs} + + + +set fd [open test$cnt.sql w] +puts $fd "BEGIN;" +puts $fd "CREATE TABLE t2(a INTEGER, b INTEGER, c VARCHAR(100));" +for {set i 1} {$i<=25000} {incr i} { + set r [expr {int(rand()*500000)}] + puts $fd "INSERT INTO t2 VALUES($i,$r,'[number_name $r]');" +} +puts $fd "COMMIT;" +close $fd +runtest {25000 INSERTs in a transaction} + + + +set fd [open test$cnt.sql w] +for {set i 0} {$i<100} {incr i} { + set lwr [expr {$i*100}] + set upr [expr {($i+10)*100}] + puts $fd "SELECT count(*), avg(b) FROM t2 WHERE b>=$lwr AND b<$upr;" +} +close $fd +runtest {100 SELECTs without an index} + + + +set fd [open test$cnt.sql w] +for {set i 1} {$i<=100} {incr i} { + puts $fd "SELECT count(*), avg(b) FROM t2 WHERE c LIKE '%[number_name $i]%';" +} +close $fd +runtest {100 SELECTs on a string comparison} + + + +set fd [open test$cnt.sql w] +puts $fd {CREATE INDEX i2a ON t2(a);} +puts $fd {CREATE INDEX i2b ON t2(b);} +close $fd +runtest {Creating an index} + + + +set fd [open test$cnt.sql w] +for {set i 0} {$i<5000} {incr i} { + set lwr [expr {$i*100}] + set upr [expr {($i+1)*100}] + puts $fd "SELECT count(*), avg(b) FROM t2 WHERE b>=$lwr AND b<$upr;" +} +close $fd +runtest {5000 SELECTs with an index} + + + +set fd [open test$cnt.sql w] +puts $fd "BEGIN;" +for {set i 0} {$i<1000} {incr i} { + set lwr [expr {$i*10}] + set upr [expr {($i+1)*10}] + puts $fd "UPDATE t1 SET b=b*2 WHERE a>=$lwr AND a<$upr;" +} +puts $fd "COMMIT;" +close $fd +runtest {1000 UPDATEs without an index} + + + +set fd [open test$cnt.sql w] +puts $fd "BEGIN;" +for {set i 1} {$i<=25000} {incr i} { + set r [expr {int(rand()*500000)}] + puts $fd "UPDATE t2 SET b=$r WHERE a=$i;" +} +puts $fd "COMMIT;" +close $fd +runtest {25000 UPDATEs with an index} + + +set fd [open test$cnt.sql w] +puts $fd "BEGIN;" +for {set i 1} {$i<=25000} {incr i} { + set r [expr {int(rand()*500000)}] + puts $fd "UPDATE t2 SET c='[number_name $r]' WHERE a=$i;" +} +puts $fd "COMMIT;" +close $fd +runtest {25000 text UPDATEs with an index} + + + +set fd [open test$cnt.sql w] +puts $fd "BEGIN;" +puts $fd "INSERT INTO t1 SELECT * FROM t2;" +puts $fd "INSERT INTO t2 SELECT * FROM t1;" +puts $fd "COMMIT;" +close $fd +runtest {INSERTs from a SELECT} + + + +set fd [open test$cnt.sql w] +puts $fd {DELETE FROM t2 WHERE c LIKE '%fifty%';} +close $fd +runtest {DELETE without an index} + + + +set fd [open test$cnt.sql w] +puts $fd {DELETE FROM t2 WHERE a>10 AND a<20000;} +close $fd +runtest {DELETE with an index} + + + +set fd [open test$cnt.sql w] +puts $fd {INSERT INTO t2 SELECT * FROM t1;} +close $fd +runtest {A big INSERT after a big DELETE} + + + +set fd [open test$cnt.sql w] +puts $fd {BEGIN;} +puts $fd {DELETE FROM t1;} +for {set i 1} {$i<=3000} {incr i} { + set r [expr {int(rand()*100000)}] + puts $fd "INSERT INTO t1 VALUES($i,$r,'[number_name $r]');" +} +puts $fd {COMMIT;} +close $fd +runtest {A big DELETE followed by many small INSERTs} + + + +set fd [open test$cnt.sql w] +puts $fd {DROP TABLE t1;} +puts $fd {DROP TABLE t2;} +close $fd +runtest {DROP TABLE} diff --git a/tool/speedtest16.c b/tool/speedtest16.c new file mode 100644 index 0000000..993cc19 --- /dev/null +++ b/tool/speedtest16.c @@ -0,0 +1,171 @@ +/* +** Performance test for SQLite. +** +** This program reads ASCII text from a file named on the command-line. +** It converts each SQL statement into UTF16 and submits it to SQLite +** for evaluation. A new UTF16 database is created at the beginning of +** the program. All statements are timed using the high-resolution timer +** built into Intel-class processors. +** +** To compile this program, first compile the SQLite library separately +** will full optimizations. For example: +** +** gcc -c -O6 -DSQLITE_THREADSAFE=0 sqlite3.c +** +** Then link against this program. But to do optimize this program +** because that defeats the hi-res timer. +** +** gcc speedtest16.c sqlite3.o -ldl -I../src +** +** Then run this program with a single argument which is the name of +** a file containing SQL script that you want to test: +** +** ./a.out database.db test.sql +*/ +#include <stdio.h> +#include <string.h> +#include <stdlib.h> +#include <ctype.h> +#include <unistd.h> +#include "sqlite3.h" + +#define ISSPACE(X) isspace((unsigned char)(X)) + +/* +** hwtime.h contains inline assembler code for implementing +** high-performance timing routines. +*/ +#include "hwtime.h" + +/* +** Convert a zero-terminated ASCII string into a zero-terminated +** UTF-16le string. Memory to hold the returned string comes +** from malloc() and should be freed by the caller. +*/ +static void *asciiToUtf16le(const char *z){ + int n = strlen(z); + char *z16; + int i, j; + + z16 = malloc( n*2 + 2 ); + for(i=j=0; i<=n; i++){ + z16[j++] = z[i]; + z16[j++] = 0; + } + return (void*)z16; +} + +/* +** Timers +*/ +static sqlite_uint64 prepTime = 0; +static sqlite_uint64 runTime = 0; +static sqlite_uint64 finalizeTime = 0; + +/* +** Prepare and run a single statement of SQL. +*/ +static void prepareAndRun(sqlite3 *db, const char *zSql){ + void *utf16; + sqlite3_stmt *pStmt; + const void *stmtTail; + sqlite_uint64 iStart, iElapse; + int rc; + + printf("****************************************************************\n"); + printf("SQL statement: [%s]\n", zSql); + utf16 = asciiToUtf16le(zSql); + iStart = sqlite3Hwtime(); + rc = sqlite3_prepare16_v2(db, utf16, -1, &pStmt, &stmtTail); + iElapse = sqlite3Hwtime() - iStart; + prepTime += iElapse; + printf("sqlite3_prepare16_v2() returns %d in %llu cycles\n", rc, iElapse); + if( rc==SQLITE_OK ){ + int nRow = 0; + iStart = sqlite3Hwtime(); + while( (rc=sqlite3_step(pStmt))==SQLITE_ROW ){ nRow++; } + iElapse = sqlite3Hwtime() - iStart; + runTime += iElapse; + printf("sqlite3_step() returns %d after %d rows in %llu cycles\n", + rc, nRow, iElapse); + iStart = sqlite3Hwtime(); + rc = sqlite3_finalize(pStmt); + iElapse = sqlite3Hwtime() - iStart; + finalizeTime += iElapse; + printf("sqlite3_finalize() returns %d in %llu cycles\n", rc, iElapse); + } + free(utf16); +} + +int main(int argc, char **argv){ + void *utf16; + sqlite3 *db; + int rc; + int nSql; + char *zSql; + int i, j; + FILE *in; + sqlite_uint64 iStart, iElapse; + sqlite_uint64 iSetup = 0; + int nStmt = 0; + int nByte = 0; + + if( argc!=3 ){ + fprintf(stderr, "Usage: %s FILENAME SQL-SCRIPT\n" + "Runs SQL-SCRIPT as UTF16 against a UTF16 database\n", + argv[0]); + exit(1); + } + in = fopen(argv[2], "r"); + fseek(in, 0L, SEEK_END); + nSql = ftell(in); + zSql = malloc( nSql+1 ); + fseek(in, 0L, SEEK_SET); + nSql = fread(zSql, 1, nSql, in); + zSql[nSql] = 0; + + printf("SQLite version: %d\n", sqlite3_libversion_number()); + unlink(argv[1]); + utf16 = asciiToUtf16le(argv[1]); + iStart = sqlite3Hwtime(); + rc = sqlite3_open16(utf16, &db); + iElapse = sqlite3Hwtime() - iStart; + iSetup = iElapse; + printf("sqlite3_open16() returns %d in %llu cycles\n", rc, iElapse); + free(utf16); + for(i=j=0; j<nSql; j++){ + if( zSql[j]==';' ){ + int isComplete; + char c = zSql[j+1]; + zSql[j+1] = 0; + isComplete = sqlite3_complete(&zSql[i]); + zSql[j+1] = c; + if( isComplete ){ + zSql[j] = 0; + while( i<j && ISSPACE(zSql[i]) ){ i++; } + if( i<j ){ + nStmt++; + nByte += j-i; + prepareAndRun(db, &zSql[i]); + } + zSql[j] = ';'; + i = j+1; + } + } + } + iStart = sqlite3Hwtime(); + sqlite3_close(db); + iElapse = sqlite3Hwtime() - iStart; + iSetup += iElapse; + printf("sqlite3_close() returns in %llu cycles\n", iElapse); + printf("\n"); + printf("Statements run: %15d\n", nStmt); + printf("Bytes of SQL text: %15d\n", nByte); + printf("Total prepare time: %15llu cycles\n", prepTime); + printf("Total run time: %15llu cycles\n", runTime); + printf("Total finalize time: %15llu cycles\n", finalizeTime); + printf("Open/Close time: %15llu cycles\n", iSetup); + printf("Total Time: %15llu cycles\n", + prepTime + runTime + finalizeTime + iSetup); + return 0; +} diff --git a/tool/speedtest2.tcl b/tool/speedtest2.tcl new file mode 100644 index 0000000..4fd632d --- /dev/null +++ b/tool/speedtest2.tcl @@ -0,0 +1,207 @@ +#!/usr/bin/tclsh +# +# Run this script using TCLSH to do a speed comparison between +# various versions of SQLite and PostgreSQL and MySQL +# + +# Run a test +# +set cnt 1 +proc runtest {title} { + global cnt + set sqlfile test$cnt.sql + puts "<h2>Test $cnt: $title</h2>" + incr cnt + set fd [open $sqlfile r] + set sql [string trim [read $fd [file size $sqlfile]]] + close $fd + set sx [split $sql \n] + set n [llength $sx] + if {$n>8} { + set sql {} + for {set i 0} {$i<3} {incr i} {append sql [lindex $sx $i]<br>\n} + append sql "<i>... [expr {$n-6}] lines omitted</i><br>\n" + for {set i [expr {$n-3}]} {$i<$n} {incr i} { + append sql [lindex $sx $i]<br>\n + } + } else { + regsub -all \n [string trim $sql] <br> sql + } + puts "<blockquote>" + puts "$sql" + puts "</blockquote><table border=0 cellpadding=0 cellspacing=0>" + set format {<tr><td>%s</td><td align="right"> %.3f</td></tr>} + set delay 1000 + exec sync; after $delay; + set t [time "exec psql drh <$sqlfile" 1] + set t [expr {[lindex $t 0]/1000000.0}] + puts [format $format PostgreSQL: $t] + exec sync; after $delay; + set t [time "exec mysql -f drh <$sqlfile" 1] + set t [expr {[lindex $t 0]/1000000.0}] + puts [format $format MySQL: $t] +# set t [time "exec ./sqlite232 s232.db <$sqlfile" 1] +# set t [expr {[lindex $t 0]/1000000.0}] +# puts [format $format {SQLite 2.3.2:} $t] +# set t [time "exec ./sqlite-100 s100.db <$sqlfile" 1] +# set t [expr {[lindex $t 0]/1000000.0}] +# puts [format $format {SQLite 2.4 (cache=100):} $t] + exec sync; after $delay; + set t [time "exec ./sqlite240 s2k.db <$sqlfile" 1] + set t [expr {[lindex $t 0]/1000000.0}] + puts [format $format {SQLite 2.4:} $t] + exec sync; after $delay; + set t [time "exec ./sqlite240 sns.db <$sqlfile" 1] + set t [expr {[lindex $t 0]/1000000.0}] + puts [format $format {SQLite 2.4 (nosync):} $t] +# set t [time "exec ./sqlite-t1 st1.db <$sqlfile" 1] +# set t [expr {[lindex $t 0]/1000000.0}] +# puts [format $format {SQLite 2.4 (test):} $t] + puts "</table>" +} + +# Initialize the environment +# +expr srand(1) +catch {exec /bin/sh -c {rm -f s*.db}} +set fd [open clear.sql w] +puts $fd { + drop table t1; + drop table t2; +} +close $fd +catch {exec psql drh <clear.sql} +catch {exec mysql drh <clear.sql} +set fd [open 2kinit.sql w] +puts $fd { + PRAGMA default_cache_size=2000; + PRAGMA default_synchronous=on; +} +close $fd +exec ./sqlite240 s2k.db <2kinit.sql +exec ./sqlite-t1 st1.db <2kinit.sql +set fd [open nosync-init.sql w] +puts $fd { + PRAGMA default_cache_size=2000; + PRAGMA default_synchronous=off; +} +close $fd +exec ./sqlite240 sns.db <nosync-init.sql +set ones {zero one two three four five six seven eight nine + ten eleven twelve thirteen fourteen fifteen sixteen seventeen + eighteen nineteen} +set tens {{} ten twenty thirty forty fifty sixty seventy eighty ninety} +proc number_name {n} { + if {$n>=1000} { + set txt "[number_name [expr {$n/1000}]] thousand" + set n [expr {$n%1000}] + } else { + set txt {} + } + if {$n>=100} { + append txt " [lindex $::ones [expr {$n/100}]] hundred" + set n [expr {$n%100}] + } + if {$n>=20} { + append txt " [lindex $::tens [expr {$n/10}]]" + set n [expr {$n%10}] + } + if {$n>0} { + append txt " [lindex $::ones $n]" + } + set txt [string trim $txt] + if {$txt==""} {set txt zero} + return $txt +} + + +set fd [open test$cnt.sql w] +puts $fd "BEGIN;" +puts $fd "CREATE TABLE t1(a INTEGER, b INTEGER, c VARCHAR(100));" +for {set i 1} {$i<=25000} {incr i} { + set r [expr {int(rand()*500000)}] + puts $fd "INSERT INTO t1 VALUES($i,$r,'[number_name $r]');" +} +puts $fd "COMMIT;" +close $fd +runtest {25000 INSERTs in a transaction} + + +set fd [open test$cnt.sql w] +puts $fd "DELETE FROM t1;" +close $fd +runtest {DELETE everything} + + +set fd [open test$cnt.sql w] +puts $fd "BEGIN;" +for {set i 1} {$i<=25000} {incr i} { + set r [expr {int(rand()*500000)}] + puts $fd "INSERT INTO t1 VALUES($i,$r,'[number_name $r]');" +} +puts $fd "COMMIT;" +close $fd +runtest {25000 INSERTs in a transaction} + + +set fd [open test$cnt.sql w] +puts $fd "DELETE FROM t1;" +close $fd +runtest {DELETE everything} + + +set fd [open test$cnt.sql w] +puts $fd "BEGIN;" +for {set i 1} {$i<=25000} {incr i} { + set r [expr {int(rand()*500000)}] + puts $fd "INSERT INTO t1 VALUES($i,$r,'[number_name $r]');" +} +puts $fd "COMMIT;" +close $fd +runtest {25000 INSERTs in a transaction} + + +set fd [open test$cnt.sql w] +puts $fd "DELETE FROM t1;" +close $fd +runtest {DELETE everything} + + +set fd [open test$cnt.sql w] +puts $fd "BEGIN;" +for {set i 1} {$i<=25000} {incr i} { + set r [expr {int(rand()*500000)}] + puts $fd "INSERT INTO t1 VALUES($i,$r,'[number_name $r]');" +} +puts $fd "COMMIT;" +close $fd +runtest {25000 INSERTs in a transaction} + + +set fd [open test$cnt.sql w] +puts $fd "DELETE FROM t1;" +close $fd +runtest {DELETE everything} + + +set fd [open test$cnt.sql w] +puts $fd "BEGIN;" +for {set i 1} {$i<=25000} {incr i} { + set r [expr {int(rand()*500000)}] + puts $fd "INSERT INTO t1 VALUES($i,$r,'[number_name $r]');" +} +puts $fd "COMMIT;" +close $fd +runtest {25000 INSERTs in a transaction} + + +set fd [open test$cnt.sql w] +puts $fd "DELETE FROM t1;" +close $fd +runtest {DELETE everything} + + +set fd [open test$cnt.sql w] +puts $fd {DROP TABLE t1;} +close $fd +runtest {DROP TABLE} diff --git a/tool/speedtest8.c b/tool/speedtest8.c new file mode 100644 index 0000000..051fc89 --- /dev/null +++ b/tool/speedtest8.c @@ -0,0 +1,260 @@ +/* +** Performance test for SQLite. +** +** This program reads ASCII text from a file named on the command-line +** and submits that text to SQLite for evaluation. A new database +** is created at the beginning of the program. All statements are +** timed using the high-resolution timer built into Intel-class processors. +** +** To compile this program, first compile the SQLite library separately +** will full optimizations. For example: +** +** gcc -c -O6 -DSQLITE_THREADSAFE=0 sqlite3.c +** +** Then link against this program. But to do optimize this program +** because that defeats the hi-res timer. +** +** gcc speedtest8.c sqlite3.o -ldl -I../src +** +** Then run this program with a single argument which is the name of +** a file containing SQL script that you want to test: +** +** ./a.out test.db test.sql +*/ +#include <stdio.h> +#include <string.h> +#include <stdlib.h> +#include <ctype.h> +#include <time.h> + +#if defined(_MSC_VER) +#include <windows.h> +#else +#include <unistd.h> +#include <sys/times.h> +#include <sched.h> +#endif + +#include "sqlite3.h" + +/* +** hwtime.h contains inline assembler code for implementing +** high-performance timing routines. +*/ +#include "hwtime.h" + +/* +** Timers +*/ +static sqlite_uint64 prepTime = 0; +static sqlite_uint64 runTime = 0; +static sqlite_uint64 finalizeTime = 0; + +/* +** Prepare and run a single statement of SQL. +*/ +static void prepareAndRun(sqlite3 *db, const char *zSql, int bQuiet){ + sqlite3_stmt *pStmt; + const char *stmtTail; + sqlite_uint64 iStart, iElapse; + int rc; + + if (!bQuiet){ + printf("***************************************************************\n"); + } + if (!bQuiet) printf("SQL statement: [%s]\n", zSql); + iStart = sqlite3Hwtime(); + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &stmtTail); + iElapse = sqlite3Hwtime() - iStart; + prepTime += iElapse; + if (!bQuiet){ + printf("sqlite3_prepare_v2() returns %d in %llu cycles\n", rc, iElapse); + } + if( rc==SQLITE_OK ){ + int nRow = 0; + iStart = sqlite3Hwtime(); + while( (rc=sqlite3_step(pStmt))==SQLITE_ROW ){ nRow++; } + iElapse = sqlite3Hwtime() - iStart; + runTime += iElapse; + if (!bQuiet){ + printf("sqlite3_step() returns %d after %d rows in %llu cycles\n", + rc, nRow, iElapse); + } + iStart = sqlite3Hwtime(); + rc = sqlite3_finalize(pStmt); + iElapse = sqlite3Hwtime() - iStart; + finalizeTime += iElapse; + if (!bQuiet){ + printf("sqlite3_finalize() returns %d in %llu cycles\n", rc, iElapse); + } + } +} + +int main(int argc, char **argv){ + sqlite3 *db; + int rc; + int nSql; + char *zSql; + int i, j; + FILE *in; + sqlite_uint64 iStart, iElapse; + sqlite_uint64 iSetup = 0; + int nStmt = 0; + int nByte = 0; + const char *zArgv0 = argv[0]; + int bQuiet = 0; +#if !defined(_MSC_VER) + struct tms tmsStart, tmsEnd; + clock_t clkStart, clkEnd; +#endif + +#ifdef HAVE_OSINST + extern sqlite3_vfs *sqlite3_instvfs_binarylog(char *, char *, char *); + extern void sqlite3_instvfs_destroy(sqlite3_vfs *); + sqlite3_vfs *pVfs = 0; +#endif + + while (argc>3) + { +#ifdef HAVE_OSINST + if( argc>4 && (strcmp(argv[1], "-log")==0) ){ + pVfs = sqlite3_instvfs_binarylog("oslog", 0, argv[2]); + sqlite3_vfs_register(pVfs, 1); + argv += 2; + argc -= 2; + continue; + } +#endif + + /* + ** Increasing the priority slightly above normal can help with + ** repeatability of testing. Note that with Cygwin, -5 equates + ** to "High", +5 equates to "Low", and anything in between + ** equates to "Normal". + */ + if( argc>4 && (strcmp(argv[1], "-priority")==0) ){ +#if defined(_MSC_VER) + int new_priority = atoi(argv[2]); + if(!SetPriorityClass(GetCurrentProcess(), + (new_priority<=-5) ? HIGH_PRIORITY_CLASS : + (new_priority<=0) ? ABOVE_NORMAL_PRIORITY_CLASS : + (new_priority==0) ? NORMAL_PRIORITY_CLASS : + (new_priority<5) ? BELOW_NORMAL_PRIORITY_CLASS : + IDLE_PRIORITY_CLASS)){ + printf ("error setting priority\n"); + exit(2); + } +#else + struct sched_param myParam; + sched_getparam(0, &myParam); + printf ("Current process priority is %d.\n", (int)myParam.sched_priority); + myParam.sched_priority = atoi(argv[2]); + printf ("Setting process priority to %d.\n", (int)myParam.sched_priority); + if (sched_setparam (0, &myParam) != 0){ + printf ("error setting priority\n"); + exit(2); + } +#endif + argv += 2; + argc -= 2; + continue; + } + + if( argc>3 && strcmp(argv[1], "-quiet")==0 ){ + bQuiet = -1; + argv++; + argc--; + continue; + } + + break; + } + + if( argc!=3 ){ + fprintf(stderr, "Usage: %s [options] FILENAME SQL-SCRIPT\n" + "Runs SQL-SCRIPT against a UTF8 database\n" + "\toptions:\n" +#ifdef HAVE_OSINST + "\t-log <log>\n" +#endif + "\t-priority <value> : set priority of task\n" + "\t-quiet : only display summary results\n", + zArgv0); + exit(1); + } + + in = fopen(argv[2], "r"); + fseek(in, 0L, SEEK_END); + nSql = ftell(in); + zSql = malloc( nSql+1 ); + fseek(in, 0L, SEEK_SET); + nSql = fread(zSql, 1, nSql, in); + zSql[nSql] = 0; + + printf("SQLite version: %d\n", sqlite3_libversion_number()); + unlink(argv[1]); +#if !defined(_MSC_VER) + clkStart = times(&tmsStart); +#endif + iStart = sqlite3Hwtime(); + rc = sqlite3_open(argv[1], &db); + iElapse = sqlite3Hwtime() - iStart; + iSetup = iElapse; + if (!bQuiet) printf("sqlite3_open() returns %d in %llu cycles\n", rc, iElapse); + for(i=j=0; j<nSql; j++){ + if( zSql[j]==';' ){ + int isComplete; + char c = zSql[j+1]; + zSql[j+1] = 0; + isComplete = sqlite3_complete(&zSql[i]); + zSql[j+1] = c; + if( isComplete ){ + zSql[j] = 0; + while( i<j && isspace(zSql[i]) ){ i++; } + if( i<j ){ + int n = j - i; + if( n>=6 && memcmp(&zSql[i], ".crash",6)==0 ) exit(1); + nStmt++; + nByte += n; + prepareAndRun(db, &zSql[i], bQuiet); + } + zSql[j] = ';'; + i = j+1; + } + } + } + iStart = sqlite3Hwtime(); + sqlite3_close(db); + iElapse = sqlite3Hwtime() - iStart; +#if !defined(_MSC_VER) + clkEnd = times(&tmsEnd); +#endif + iSetup += iElapse; + if (!bQuiet) printf("sqlite3_close() returns in %llu cycles\n", iElapse); + + printf("\n"); + printf("Statements run: %15d stmts\n", nStmt); + printf("Bytes of SQL text: %15d bytes\n", nByte); + printf("Total prepare time: %15llu cycles\n", prepTime); + printf("Total run time: %15llu cycles\n", runTime); + printf("Total finalize time: %15llu cycles\n", finalizeTime); + printf("Open/Close time: %15llu cycles\n", iSetup); + printf("Total time: %15llu cycles\n", + prepTime + runTime + finalizeTime + iSetup); + +#if !defined(_MSC_VER) + printf("\n"); + printf("Total user CPU time: %15.3g secs\n", (tmsEnd.tms_utime - tmsStart.tms_utime)/(double)CLOCKS_PER_SEC ); + printf("Total system CPU time: %15.3g secs\n", (tmsEnd.tms_stime - tmsStart.tms_stime)/(double)CLOCKS_PER_SEC ); + printf("Total real time: %15.3g secs\n", (clkEnd -clkStart)/(double)CLOCKS_PER_SEC ); +#endif + +#ifdef HAVE_OSINST + if( pVfs ){ + sqlite3_instvfs_destroy(pVfs); + printf("vfs log written to %s\n", argv[0]); + } +#endif + + return 0; +} diff --git a/tool/speedtest8inst1.c b/tool/speedtest8inst1.c new file mode 100644 index 0000000..ceaeca0 --- /dev/null +++ b/tool/speedtest8inst1.c @@ -0,0 +1,218 @@ +/* +** Performance test for SQLite. +** +** This program reads ASCII text from a file named on the command-line +** and submits that text to SQLite for evaluation. A new database +** is created at the beginning of the program. All statements are +** timed using the high-resolution timer built into Intel-class processors. +** +** To compile this program, first compile the SQLite library separately +** will full optimizations. For example: +** +** gcc -c -O6 -DSQLITE_THREADSAFE=0 sqlite3.c +** +** Then link against this program. But to do optimize this program +** because that defeats the hi-res timer. +** +** gcc speedtest8.c sqlite3.o -ldl -I../src +** +** Then run this program with a single argument which is the name of +** a file containing SQL script that you want to test: +** +** ./a.out test.db test.sql +*/ +#include <stdio.h> +#include <string.h> +#include <stdlib.h> +#include <ctype.h> +#include <unistd.h> +#include <stdarg.h> +#include "sqlite3.h" + +#define ISSPACE(X) isspace((unsigned char)(X)) + +#include "test_osinst.c" + +/* +** Prepare and run a single statement of SQL. +*/ +static void prepareAndRun(sqlite3_vfs *pInstVfs, sqlite3 *db, const char *zSql){ + sqlite3_stmt *pStmt; + const char *stmtTail; + int rc; + char zMessage[1024]; + zMessage[1023] = '\0'; + + sqlite3_uint64 iTime; + + sqlite3_snprintf(1023, zMessage, "sqlite3_prepare_v2: %s", zSql); + sqlite3_instvfs_binarylog_marker(pInstVfs, zMessage); + + iTime = sqlite3Hwtime(); + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &stmtTail); + iTime = sqlite3Hwtime() - iTime; + sqlite3_instvfs_binarylog_call(pInstVfs,BINARYLOG_PREPARE_V2,iTime,rc,zSql); + + if( rc==SQLITE_OK ){ + int nRow = 0; + + sqlite3_snprintf(1023, zMessage, "sqlite3_step loop: %s", zSql); + sqlite3_instvfs_binarylog_marker(pInstVfs, zMessage); + iTime = sqlite3Hwtime(); + while( (rc=sqlite3_step(pStmt))==SQLITE_ROW ){ nRow++; } + iTime = sqlite3Hwtime() - iTime; + sqlite3_instvfs_binarylog_call(pInstVfs, BINARYLOG_STEP, iTime, rc, zSql); + + sqlite3_snprintf(1023, zMessage, "sqlite3_finalize: %s", zSql); + sqlite3_instvfs_binarylog_marker(pInstVfs, zMessage); + iTime = sqlite3Hwtime(); + rc = sqlite3_finalize(pStmt); + iTime = sqlite3Hwtime() - iTime; + sqlite3_instvfs_binarylog_call(pInstVfs, BINARYLOG_FINALIZE, iTime, rc, zSql); + } +} + +static int stringcompare(const char *zLeft, const char *zRight){ + int ii; + for(ii=0; zLeft[ii] && zRight[ii]; ii++){ + if( zLeft[ii]!=zRight[ii] ) return 0; + } + return( zLeft[ii]==zRight[ii] ); +} + +static char *readScriptFile(const char *zFile, int *pnScript){ + sqlite3_vfs *pVfs = sqlite3_vfs_find(0); + sqlite3_file *p; + int rc; + sqlite3_int64 nByte; + char *zData = 0; + int flags = SQLITE_OPEN_READONLY|SQLITE_OPEN_MAIN_DB; + + p = (sqlite3_file *)malloc(pVfs->szOsFile); + rc = pVfs->xOpen(pVfs, zFile, p, flags, &flags); + if( rc!=SQLITE_OK ){ + goto error_out; + } + + rc = p->pMethods->xFileSize(p, &nByte); + if( rc!=SQLITE_OK ){ + goto close_out; + } + + zData = (char *)malloc(nByte+1); + rc = p->pMethods->xRead(p, zData, nByte, 0); + if( rc!=SQLITE_OK ){ + goto close_out; + } + zData[nByte] = '\0'; + + p->pMethods->xClose(p); + free(p); + *pnScript = nByte; + return zData; + +close_out: + p->pMethods->xClose(p); + +error_out: + free(p); + free(zData); + return 0; +} + +int main(int argc, char **argv){ + + const char zUsageMsg[] = + "Usage: %s options...\n" + " where available options are:\n" + "\n" + " -db DATABASE-FILE (database file to operate on)\n" + " -script SCRIPT-FILE (script file to read sql from)\n" + " -log LOG-FILE (log file to create)\n" + " -logdata (log all data to log file)\n" + "\n" + " Options -db, -script and -log are compulsory\n" + "\n" + ; + + const char *zDb = 0; + const char *zScript = 0; + const char *zLog = 0; + int logdata = 0; + + int ii; + int i, j; + int rc; + + sqlite3_vfs *pInstVfs; /* Instrumentation VFS */ + + char *zSql = 0; + int nSql; + + sqlite3 *db; + + for(ii=1; ii<argc; ii++){ + if( stringcompare("-db", argv[ii]) && (ii+1)<argc ){ + zDb = argv[++ii]; + } + + else if( stringcompare("-script", argv[ii]) && (ii+1)<argc ){ + zScript = argv[++ii]; + } + + else if( stringcompare("-log", argv[ii]) && (ii+1)<argc ){ + zLog = argv[++ii]; + } + + else if( stringcompare("-logdata", argv[ii]) ){ + logdata = 1; + } + + else { + goto usage; + } + } + if( !zDb || !zScript || !zLog ) goto usage; + + zSql = readScriptFile(zScript, &nSql); + if( !zSql ){ + fprintf(stderr, "Failed to read script file\n"); + return -1; + } + + pInstVfs = sqlite3_instvfs_binarylog("logging", 0, zLog, logdata); + + rc = sqlite3_open_v2( + zDb, &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, "logging" + ); + if( rc!=SQLITE_OK ){ + fprintf(stderr, "Failed to open db: %s\n", sqlite3_errmsg(db)); + return -2; + } + + for(i=j=0; j<nSql; j++){ + if( zSql[j]==';' ){ + int isComplete; + char c = zSql[j+1]; + zSql[j+1] = 0; + isComplete = sqlite3_complete(&zSql[i]); + zSql[j+1] = c; + if( isComplete ){ + zSql[j] = 0; + while( i<j && ISSPACE(zSql[i]) ){ i++; } + if( i<j ){ + prepareAndRun(pInstVfs, db, &zSql[i]); + } + zSql[j] = ';'; + i = j+1; + } + } + } + + sqlite3_instvfs_destroy(pInstVfs); + return 0; + +usage: + fprintf(stderr, zUsageMsg, argv[0]); + return -3; +} diff --git a/tool/spellsift.tcl b/tool/spellsift.tcl new file mode 100755 index 0000000..4e67c3e --- /dev/null +++ b/tool/spellsift.tcl @@ -0,0 +1,74 @@ +#!/usr/bin/tclsh + +set usage { + Usage: spellsift.tcl <source_filenames> + The named .c and .h source files comment blocks are spell-checked. +} + +if {[llength $argv] == 0} { + puts stderr $usage + exit 0 +} + +# Want a Tcl version with 3-argument close. +package require Tcl 8.6 + +set ::spellchk "aspell --extra-dicts ./custom.rws list" + +# Run text through aspell with custom dictionary, return finds. +proc misspelled {text} { + set spellerr [open "|$::spellchk" r+] + puts $spellerr $text + flush $spellerr + close $spellerr write + set huhq [regsub {\s*$} [read $spellerr] {}] + close $spellerr read + return [split $huhq "\n"] +} + +# Eliminate some common patterns that need not be well spelled. +proc decruft {text} { + set nopp [regsub -all "\n *#\[^\n\]*\n" $text "\n\n" ] + set noticket [regsub -all {Ticket \[?[0-9a-f]+\]?} $nopp "" ] + return $noticket +} + +# Sift out common variable spellings not in normal dictionaries. +proc varsift {words} { + set rv [list] + foreach w $words { + set n [string length $w] + set cr [string range $w 1 end] + if {[string tolower $cr] ne $cr} continue + lappend rv $w; + } + return $rv +} + +foreach fname $argv { + set ich [open $fname r] + set dtext [decruft [read $ich]] + close $ich + set cbounds [regexp -indices -inline -all {(/\*)|(\*/)} $dtext] + set ccb -1 + set cblocks [list] + foreach {ap cb ce} $cbounds { + set cib [lindex $cb 1] + set cie [lindex $ce 0] + if {$cie != -1} { + if {$ccb != -1} { + set cce [expr $cie - 1] + set destar [string map [list * " "] [string range $dtext $ccb $cce]] + lappend cblocks $destar + set ccb -1 + } else continue + } elseif {$cib != -1} { + set ccb [expr $cib + 1] + } + } + set oddspells [varsift [misspelled [join $cblocks "\n"]]] + if {[llength $oddspells] > 0} { + puts "!? Misspellings from $fname:" + puts [join [lsort -nocase -unique $oddspells] "\n"] + } +} diff --git a/tool/split-sqlite3c.tcl b/tool/split-sqlite3c.tcl new file mode 100644 index 0000000..0308431 --- /dev/null +++ b/tool/split-sqlite3c.tcl @@ -0,0 +1,96 @@ +#!/usr/bin/tclsh +# +# This script splits the sqlite3.c amalgamated source code files into +# several smaller files such that no single files is more than a fixed +# number of lines in length (32k or 64k). Each of the split out files +# is #include-ed by the master file. +# +# Splitting files up this way allows them to be used with older compilers +# that cannot handle really long source files. +# +set MAX 32768 ;# Maximum number of lines per file. + +set BEGIN {^/\*+ Begin file ([a-zA-Z0-9_.]+) \*+/} +set END {^/\*+ End of %s \*+/} + +set in [open sqlite3.c] +set out1 [open sqlite3-all.c w] +fconfigure $out1 -translation lf + +# Copy the header from sqlite3.c into sqlite3-all.c +# +while {[gets $in line]} { + if {[regexp $BEGIN $line]} break + puts $out1 $line +} + +# Gather the complete content of a file into memory. Store the +# content in $bufout. Store the number of lines is $nout +# +proc gather_one_file {firstline bufout nout} { + regexp $::BEGIN $firstline all filename + set end [format $::END $filename] + upvar $bufout buf $nout n + set buf $firstline\n + global in + set n 0 + while {[gets $in line]>=0} { + incr n + append buf $line\n + if {[regexp $end $line]} break + } +} + +# Write a big chunk of text in to an auxiliary file "sqlite3-NNN.c". +# Also add an appropriate #include to sqlite3-all.c +# +set filecnt 0 +proc write_one_file {content} { + global filecnt + incr filecnt + set label $filecnt + if {$filecnt>9} { + set label [string index ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnop \ + [expr {$filecnt-10}]] + } else { + set label $filecnt + } + set out [open sqlite3-$label.c w] + fconfigure $out -translation lf + puts -nonewline $out $content + close $out + puts $::out1 "#include \"sqlite3-$filecnt.c\"" +} + +# Continue reading input. Store chunks in separate files and add +# the #includes to the main sqlite3-all.c file as necessary to reference +# the extra chunks. +# +set all {} +set N 0 +while {[regexp $BEGIN $line]} { + set buf {} + set n 0 + gather_one_file $line buf n + if {$n+$N>=$MAX} { + write_one_file $all + set all {} + set N 0 + } + append all $buf + incr N $n + while {[gets $in line]>=0} { + if {[regexp $BEGIN $line]} break + if {$N>0} { + write_one_file $all + set N 0 + set all {} + } + puts $out1 $line + } +} +if {$N>0} { + write_one_file $all +} +close $out1 +close $in diff --git a/tool/sqldiff.c b/tool/sqldiff.c new file mode 100644 index 0000000..cbdfc35 --- /dev/null +++ b/tool/sqldiff.c @@ -0,0 +1,2055 @@ +/* +** 2015-04-06 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This is a utility program that computes the differences in content +** between two SQLite databases. +** +** To compile, simply link against SQLite. (Windows builds must also link +** against ext/consio/console_io.c.) +** +** See the showHelp() routine below for a brief description of how to +** run the utility. +*/ +#include <stdio.h> +#include <stdlib.h> +#include <stdarg.h> +#include <ctype.h> +#include <string.h> +#include <assert.h> +#include "sqlite3.h" + +/* Output function substitutions that cause UTF8 characters to be rendered +** correctly on Windows: +** +** fprintf() -> Wfprintf() +** +*/ +#if defined(_WIN32) +# include "console_io.h" +# define Wfprintf fPrintfUtf8 +#else +# define Wfprintf fprintf +#endif + +/* +** All global variables are gathered into the "g" singleton. +*/ +struct GlobalVars { + const char *zArgv0; /* Name of program */ + int bSchemaOnly; /* Only show schema differences */ + int bSchemaPK; /* Use the schema-defined PK, not the true PK */ + int bHandleVtab; /* Handle fts3, fts4, fts5 and rtree vtabs */ + unsigned fDebug; /* Debug flags */ + int bSchemaCompare; /* Doing single-table sqlite_schema compare */ + sqlite3 *db; /* The database connection */ +} g; + +/* +** Allowed values for g.fDebug +*/ +#define DEBUG_COLUMN_NAMES 0x000001 +#define DEBUG_DIFF_SQL 0x000002 + +/* +** Clear and free an sqlite3_str object +*/ +static void strFree(sqlite3_str *pStr){ + sqlite3_free(sqlite3_str_finish(pStr)); +} + +/* +** Print an error resulting from faulting command-line arguments and +** abort the program. +*/ +static void cmdlineError(const char *zFormat, ...){ + sqlite3_str *pOut = sqlite3_str_new(0); + va_list ap; + va_start(ap, zFormat); + sqlite3_str_vappendf(pOut, zFormat, ap); + va_end(ap); + Wfprintf(stderr, "%s: %s\n", g.zArgv0, sqlite3_str_value(pOut)); + strFree(pOut); + Wfprintf(stderr, "\"%s --help\" for more help\n", g.zArgv0); + exit(1); +} + +/* +** Print an error message for an error that occurs at runtime, then +** abort the program. +*/ +static void runtimeError(const char *zFormat, ...){ + sqlite3_str *pOut = sqlite3_str_new(0); + va_list ap; + va_start(ap, zFormat); + sqlite3_str_vappendf(pOut, zFormat, ap); + va_end(ap); + Wfprintf(stderr, "%s: %s\n", g.zArgv0, sqlite3_str_value(pOut)); + strFree(pOut); + exit(1); +} + + +/* Safely quote an SQL identifier. Use the minimum amount of transformation +** necessary to allow the string to be used with %s. +** +** Space to hold the returned string is obtained from sqlite3_malloc(). The +** caller is responsible for ensuring this space is freed when no longer +** needed. +*/ +static char *safeId(const char *zId){ + int i, x; + char c; + if( zId[0]==0 ) return sqlite3_mprintf("\"\""); + for(i=x=0; (c = zId[i])!=0; i++){ + if( !isalpha(c) && c!='_' ){ + if( i>0 && isdigit(c) ){ + x++; + }else{ + return sqlite3_mprintf("\"%w\"", zId); + } + } + } + if( x || !sqlite3_keyword_check(zId,i) ){ + return sqlite3_mprintf("%s", zId); + } + return sqlite3_mprintf("\"%w\"", zId); +} + +/* +** Prepare a new SQL statement. Print an error and abort if anything +** goes wrong. +*/ +static sqlite3_stmt *db_vprepare(const char *zFormat, va_list ap){ + char *zSql; + int rc; + sqlite3_stmt *pStmt; + + zSql = sqlite3_vmprintf(zFormat, ap); + if( zSql==0 ) runtimeError("out of memory"); + rc = sqlite3_prepare_v2(g.db, zSql, -1, &pStmt, 0); + if( rc ){ + runtimeError("SQL statement error: %s\n\"%s\"", sqlite3_errmsg(g.db), + zSql); + } + sqlite3_free(zSql); + return pStmt; +} +static sqlite3_stmt *db_prepare(const char *zFormat, ...){ + va_list ap; + sqlite3_stmt *pStmt; + va_start(ap, zFormat); + pStmt = db_vprepare(zFormat, ap); + va_end(ap); + return pStmt; +} + +/* +** Free a list of strings +*/ +static void namelistFree(char **az){ + if( az ){ + int i; + for(i=0; az[i]; i++) sqlite3_free(az[i]); + sqlite3_free(az); + } +} + +/* +** Return a list of column names [a] for the table zDb.zTab. Space to +** hold the list is obtained from sqlite3_malloc() and should released +** using namelistFree() when no longer needed. +** +** Primary key columns are listed first, followed by data columns. +** The number of columns in the primary key is returned in *pnPkey. +** +** Normally [a], the "primary key" in the previous sentence is the true +** primary key - the rowid or INTEGER PRIMARY KEY for ordinary tables +** or the declared PRIMARY KEY for WITHOUT ROWID tables. However, if +** the g.bSchemaPK flag is set, then the schema-defined PRIMARY KEY is +** used in all cases. In that case, entries that have NULL values in +** any of their primary key fields will be excluded from the analysis. +** +** If the primary key for a table is the rowid but rowid is inaccessible, +** then this routine returns a NULL pointer. +** +** [a. If the lone, named table is "sqlite_schema", "rootpage" column is +** omitted and the "type" and "name" columns are made to be the PK.] +** +** Examples: +** CREATE TABLE t1(a INT UNIQUE, b INTEGER, c TEXT, PRIMARY KEY(c)); +** *pnPKey = 1; +** az = { "rowid", "a", "b", "c", 0 } // Normal case +** az = { "c", "a", "b", 0 } // g.bSchemaPK==1 +** +** CREATE TABLE t2(a INT UNIQUE, b INTEGER, c TEXT, PRIMARY KEY(b)); +** *pnPKey = 1; +** az = { "b", "a", "c", 0 } +** +** CREATE TABLE t3(x,y,z,PRIMARY KEY(y,z)); +** *pnPKey = 1 // Normal case +** az = { "rowid", "x", "y", "z", 0 } // Normal case +** *pnPKey = 2 // g.bSchemaPK==1 +** az = { "y", "x", "z", 0 } // g.bSchemaPK==1 +** +** CREATE TABLE t4(x,y,z,PRIMARY KEY(y,z)) WITHOUT ROWID; +** *pnPKey = 2 +** az = { "y", "z", "x", 0 } +** +** CREATE TABLE t5(rowid,_rowid_,oid); +** az = 0 // The rowid is not accessible +*/ +static char **columnNames( + const char *zDb, /* Database ("main" or "aux") to query */ + const char *zTab, /* Name of table to return details of */ + int *pnPKey, /* OUT: Number of PK columns */ + int *pbRowid /* OUT: True if PK is an implicit rowid */ +){ + char **az = 0; /* List of column names to be returned */ + int naz = 0; /* Number of entries in az[] */ + sqlite3_stmt *pStmt; /* SQL statement being run */ + char *zPkIdxName = 0; /* Name of the PRIMARY KEY index */ + int truePk = 0; /* PRAGMA table_info indentifies the PK to use */ + int nPK = 0; /* Number of PRIMARY KEY columns */ + int i, j; /* Loop counters */ + + if( g.bSchemaPK==0 ){ + /* Normal case: Figure out what the true primary key is for the table. + ** * For WITHOUT ROWID tables, the true primary key is the same as + ** the schema PRIMARY KEY, which is guaranteed to be present. + ** * For rowid tables with an INTEGER PRIMARY KEY, the true primary + ** key is the INTEGER PRIMARY KEY. + ** * For all other rowid tables, the rowid is the true primary key. + */ + pStmt = db_prepare("PRAGMA %s.index_list=%Q", zDb, zTab); + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( sqlite3_stricmp((const char*)sqlite3_column_text(pStmt,3),"pk")==0 ){ + zPkIdxName = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1)); + break; + } + } + sqlite3_finalize(pStmt); + if( zPkIdxName ){ + int nKey = 0; + int nCol = 0; + truePk = 0; + pStmt = db_prepare("PRAGMA %s.index_xinfo=%Q", zDb, zPkIdxName); + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + nCol++; + if( sqlite3_column_int(pStmt,5) ){ nKey++; continue; } + if( sqlite3_column_int(pStmt,1)>=0 ) truePk = 1; + } + if( nCol==nKey ) truePk = 1; + if( truePk ){ + nPK = nKey; + }else{ + nPK = 1; + } + sqlite3_finalize(pStmt); + sqlite3_free(zPkIdxName); + }else{ + truePk = 1; + nPK = 1; + } + pStmt = db_prepare("PRAGMA %s.table_info=%Q", zDb, zTab); + }else{ + /* The g.bSchemaPK==1 case: Use whatever primary key is declared + ** in the schema. The "rowid" will still be used as the primary key + ** if the table definition does not contain a PRIMARY KEY. + */ + nPK = 0; + pStmt = db_prepare("PRAGMA %s.table_info=%Q", zDb, zTab); + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( sqlite3_column_int(pStmt,5)>0 ) nPK++; + } + sqlite3_reset(pStmt); + if( nPK==0 ) nPK = 1; + truePk = 1; + } + if( g.bSchemaCompare ){ + assert( sqlite3_stricmp(zTab,"sqlite_schema")==0 + || sqlite3_stricmp(zTab,"sqlite_master")==0 ); + /* For sqlite_schema, will use type and name as the PK. */ + nPK = 2; + truePk = 0; + } + *pnPKey = nPK; + naz = nPK; + az = sqlite3_malloc( sizeof(char*)*(nPK+1) ); + if( az==0 ) runtimeError("out of memory"); + memset(az, 0, sizeof(char*)*(nPK+1)); + if( g.bSchemaCompare ){ + az[0] = sqlite3_mprintf("%s", "type"); + az[1] = sqlite3_mprintf("%s", "name"); + } + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + char * sid = safeId((char*)sqlite3_column_text(pStmt,1)); + int iPKey; + if( truePk && (iPKey = sqlite3_column_int(pStmt,5))>0 ){ + az[iPKey-1] = sid; + }else{ + if( !g.bSchemaCompare + || !(strcmp(sid,"rootpage")==0 + ||strcmp(sid,"name")==0 + ||strcmp(sid,"type")==0)){ + az = sqlite3_realloc(az, sizeof(char*)*(naz+2) ); + if( az==0 ) runtimeError("out of memory"); + az[naz++] = sid; + } + } + } + sqlite3_finalize(pStmt); + if( az ) az[naz] = 0; + + /* If it is non-NULL, set *pbRowid to indicate whether or not the PK of + ** this table is an implicit rowid (*pbRowid==1) or not (*pbRowid==0). */ + if( pbRowid ) *pbRowid = (az[0]==0); + + /* If this table has an implicit rowid for a PK, figure out how to refer + ** to it. There are usually three options - "rowid", "_rowid_" and "oid". + ** Any of these will work, unless the table has an explicit column of the + ** same name or the sqlite_schema tables are to be compared. In the latter + ** case, pretend that the "true" primary key is the name column, which + ** avoids extraneous diffs against the schemas due to rowid variance. */ + if( az[0]==0 ){ + const char *azRowid[] = { "rowid", "_rowid_", "oid" }; + for(i=0; i<sizeof(azRowid)/sizeof(azRowid[0]); i++){ + for(j=1; j<naz; j++){ + if( sqlite3_stricmp(az[j], azRowid[i])==0 ) break; + } + if( j>=naz ){ + az[0] = sqlite3_mprintf("%s", azRowid[i]); + break; + } + } + if( az[0]==0 ){ + for(i=1; i<naz; i++) sqlite3_free(az[i]); + sqlite3_free(az); + az = 0; + } + } + return az; +} + +/* +** Print the sqlite3_value X as an SQL literal. +*/ +static void printQuoted(FILE *out, sqlite3_value *X){ + switch( sqlite3_value_type(X) ){ + case SQLITE_FLOAT: { + double r1; + char zBuf[50]; + r1 = sqlite3_value_double(X); + sqlite3_snprintf(sizeof(zBuf), zBuf, "%!.15g", r1); + fprintf(out, "%s", zBuf); + break; + } + case SQLITE_INTEGER: { + fprintf(out, "%lld", sqlite3_value_int64(X)); + break; + } + case SQLITE_BLOB: { + const unsigned char *zBlob = sqlite3_value_blob(X); + int nBlob = sqlite3_value_bytes(X); + if( zBlob ){ + int i; + fprintf(out, "x'"); + for(i=0; i<nBlob; i++){ + fprintf(out, "%02x", zBlob[i]); + } + fprintf(out, "'"); + }else{ + /* Could be an OOM, could be a zero-byte blob */ + fprintf(out, "X''"); + } + break; + } + case SQLITE_TEXT: { + const unsigned char *zArg = sqlite3_value_text(X); + + if( zArg==0 ){ + fprintf(out, "NULL"); + }else{ + int inctl = 0; + int i, j; + fprintf(out, "'"); + for(i=j=0; zArg[i]; i++){ + char c = zArg[i]; + int ctl = iscntrl(c); + if( ctl>inctl ){ + inctl = ctl; + fprintf(out, "%.*s'||X'%02x", i-j, &zArg[j], c); + j = i+1; + }else if( ctl ){ + fprintf(out, "%02x", c); + j = i+1; + }else{ + if( inctl ){ + inctl = 0; + fprintf(out, "'\n||'"); + } + if( c=='\'' ){ + fprintf(out, "%.*s'", i-j+1, &zArg[j]); + j = i+1; + } + } + } + fprintf(out, "%s'", &zArg[j]); + } + break; + } + case SQLITE_NULL: { + fprintf(out, "NULL"); + break; + } + } +} + +/* +** Output SQL that will recreate the aux.zTab table. +*/ +static void dump_table(const char *zTab, FILE *out){ + char *zId = safeId(zTab); /* Name of the table */ + char **az = 0; /* List of columns */ + int nPk; /* Number of true primary key columns */ + int nCol; /* Number of data columns */ + int i; /* Loop counter */ + sqlite3_stmt *pStmt; /* SQL statement */ + const char *zSep; /* Separator string */ + sqlite3_str *pIns; /* Beginning of the INSERT statement */ + + pStmt = db_prepare("SELECT sql FROM aux.sqlite_schema WHERE name=%Q", zTab); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + fprintf(out, "%s;\n", sqlite3_column_text(pStmt,0)); + } + sqlite3_finalize(pStmt); + if( !g.bSchemaOnly ){ + az = columnNames("aux", zTab, &nPk, 0); + pIns = sqlite3_str_new(0); + if( az==0 ){ + pStmt = db_prepare("SELECT * FROM aux.%s", zId); + sqlite3_str_appendf(pIns,"INSERT INTO %s VALUES", zId); + }else{ + sqlite3_str *pSql = sqlite3_str_new(0); + zSep = "SELECT"; + for(i=0; az[i]; i++){ + sqlite3_str_appendf(pSql, "%s %s", zSep, az[i]); + zSep = ","; + } + sqlite3_str_appendf(pSql," FROM aux.%s", zId); + zSep = " ORDER BY"; + for(i=1; i<=nPk; i++){ + sqlite3_str_appendf(pSql, "%s %d", zSep, i); + zSep = ","; + } + pStmt = db_prepare("%s", sqlite3_str_value(pSql)); + strFree(pSql); + sqlite3_str_appendf(pIns, "INSERT INTO %s", zId); + zSep = "("; + for(i=0; az[i]; i++){ + sqlite3_str_appendf(pIns, "%s%s", zSep, az[i]); + zSep = ","; + } + sqlite3_str_appendf(pIns,") VALUES"); + namelistFree(az); + } + nCol = sqlite3_column_count(pStmt); + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + Wfprintf(out, "%s",sqlite3_str_value(pIns)); + zSep = "("; + for(i=0; i<nCol; i++){ + Wfprintf(out, "%s",zSep); + printQuoted(out, sqlite3_column_value(pStmt,i)); + zSep = ","; + } + Wfprintf(out, ");\n"); + } + sqlite3_finalize(pStmt); + strFree(pIns); + } /* endif !g.bSchemaOnly */ + pStmt = db_prepare("SELECT sql FROM aux.sqlite_schema" + " WHERE type='index' AND tbl_name=%Q AND sql IS NOT NULL", + zTab); + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + Wfprintf(out, "%s;\n", sqlite3_column_text(pStmt,0)); + } + sqlite3_finalize(pStmt); + sqlite3_free(zId); +} + + +/* +** Compute all differences for a single table, except if the +** table name is sqlite_schema, ignore the rootpage column. +*/ +static void diff_one_table(const char *zTab, FILE *out){ + char *zId = safeId(zTab); /* Name of table (translated for us in SQL) */ + char **az = 0; /* Columns in main */ + char **az2 = 0; /* Columns in aux */ + int nPk; /* Primary key columns in main */ + int nPk2; /* Primary key columns in aux */ + int n = 0; /* Number of columns in main */ + int n2; /* Number of columns in aux */ + int nQ; /* Number of output columns in the diff query */ + int i; /* Loop counter */ + const char *zSep; /* Separator string */ + sqlite3_str *pSql; /* Comparison query */ + sqlite3_stmt *pStmt; /* Query statement to do the diff */ + const char *zLead = /* Becomes line-comment for sqlite_schema */ + (g.bSchemaCompare)? "-- " : ""; + + pSql = sqlite3_str_new(0); + if( g.fDebug==DEBUG_COLUMN_NAMES ){ + /* Simply run columnNames() on all tables of the origin + ** database and show the results. This is used for testing + ** and debugging of the columnNames() function. + */ + az = columnNames("aux",zTab, &nPk, 0); + if( az==0 ){ + Wfprintf(stdout, "Rowid not accessible for %s\n", zId); + }else{ + Wfprintf(stdout, "%s:", zId); + for(i=0; az[i]; i++){ + Wfprintf(stdout, " %s", az[i]); + if( i+1==nPk ) Wfprintf(stdout, " *"); + } + Wfprintf(stdout, "\n"); + } + goto end_diff_one_table; + } + + if( sqlite3_table_column_metadata(g.db,"aux",zTab,0,0,0,0,0,0) ){ + if( !sqlite3_table_column_metadata(g.db,"main",zTab,0,0,0,0,0,0) ){ + /* Table missing from second database. */ + if( g.bSchemaCompare ) + Wfprintf(out, "-- 2nd DB has no %s table\n", zTab); + else + Wfprintf(out, "DROP TABLE %s;\n", zId); + } + goto end_diff_one_table; + } + + if( sqlite3_table_column_metadata(g.db,"main",zTab,0,0,0,0,0,0) ){ + /* Table missing from source */ + if( g.bSchemaCompare ){ + Wfprintf(out, "-- 1st DB has no %s table\n", zTab); + }else{ + dump_table(zTab, out); + } + goto end_diff_one_table; + } + + az = columnNames("main", zTab, &nPk, 0); + az2 = columnNames("aux", zTab, &nPk2, 0); + if( az && az2 ){ + for(n=0; az[n] && az2[n]; n++){ + if( sqlite3_stricmp(az[n],az2[n])!=0 ) break; + } + } + if( az==0 + || az2==0 + || nPk!=nPk2 + || az[n] + ){ + /* Schema mismatch */ + Wfprintf(out, "%sDROP TABLE %s; -- due to schema mismatch\n", zLead, zId); + dump_table(zTab, out); + goto end_diff_one_table; + } + + /* Build the comparison query */ + for(n2=n; az2[n2]; n2++){ + char *zNTab = safeId(az2[n2]); + Wfprintf(out, "ALTER TABLE %s ADD COLUMN %s;\n", zId, zNTab); + sqlite3_free(zNTab); + } + nQ = nPk2+1+2*(n2-nPk2); + if( n2>nPk2 ){ + zSep = "SELECT "; + for(i=0; i<nPk; i++){ + sqlite3_str_appendf(pSql, "%sB.%s", zSep, az[i]); + zSep = ", "; + } + sqlite3_str_appendf(pSql, ", 1 /* changed row */"); + while( az[i] ){ + sqlite3_str_appendf(pSql, ", A.%s IS NOT B.%s, B.%s", + az[i], az2[i], az2[i]); + i++; + } + while( az2[i] ){ + sqlite3_str_appendf(pSql, ", B.%s IS NOT NULL, B.%s", + az2[i], az2[i]); + i++; + } + sqlite3_str_appendf(pSql, "\n FROM main.%s A, aux.%s B\n", zId, zId); + zSep = " WHERE"; + for(i=0; i<nPk; i++){ + sqlite3_str_appendf(pSql, "%s A.%s=B.%s", zSep, az[i], az[i]); + zSep = " AND"; + } + zSep = "\n AND ("; + while( az[i] ){ + sqlite3_str_appendf(pSql, "%sA.%s IS NOT B.%s%s\n", + zSep, az[i], az2[i], az2[i+1]==0 ? ")" : ""); + zSep = " OR "; + i++; + } + while( az2[i] ){ + sqlite3_str_appendf(pSql, "%sB.%s IS NOT NULL%s\n", + zSep, az2[i], az2[i+1]==0 ? ")" : ""); + zSep = " OR "; + i++; + } + sqlite3_str_appendf(pSql, " UNION ALL\n"); + } + zSep = "SELECT "; + for(i=0; i<nPk; i++){ + sqlite3_str_appendf(pSql, "%sA.%s", zSep, az[i]); + zSep = ", "; + } + sqlite3_str_appendf(pSql, ", 2 /* deleted row */"); + while( az2[i] ){ + sqlite3_str_appendf(pSql, ", NULL, NULL"); + i++; + } + sqlite3_str_appendf(pSql, "\n FROM main.%s A\n", zId); + sqlite3_str_appendf(pSql, " WHERE NOT EXISTS(SELECT 1 FROM aux.%s B\n", zId); + zSep = " WHERE"; + for(i=0; i<nPk; i++){ + sqlite3_str_appendf(pSql, "%s A.%s=B.%s", zSep, az[i], az[i]); + zSep = " AND"; + } + sqlite3_str_appendf(pSql, ")\n"); + zSep = " UNION ALL\nSELECT "; + for(i=0; i<nPk; i++){ + sqlite3_str_appendf(pSql, "%sB.%s", zSep, az[i]); + zSep = ", "; + } + sqlite3_str_appendf(pSql, ", 3 /* inserted row */"); + while( az2[i] ){ + sqlite3_str_appendf(pSql, ", 1, B.%s", az2[i]); + i++; + } + sqlite3_str_appendf(pSql, "\n FROM aux.%s B\n", zId); + sqlite3_str_appendf(pSql, " WHERE NOT EXISTS(SELECT 1 FROM main.%s A\n", zId); + zSep = " WHERE"; + for(i=0; i<nPk; i++){ + sqlite3_str_appendf(pSql, "%s A.%s=B.%s", zSep, az[i], az[i]); + zSep = " AND"; + } + sqlite3_str_appendf(pSql, ")\n ORDER BY"); + zSep = " "; + for(i=1; i<=nPk; i++){ + sqlite3_str_appendf(pSql, "%s%d", zSep, i); + zSep = ", "; + } + sqlite3_str_appendf(pSql, ";\n"); + + if( g.fDebug & DEBUG_DIFF_SQL ){ + printf("SQL for %s:\n%s\n", zId, sqlite3_str_value(pSql)); + goto end_diff_one_table; + } + + /* Drop indexes that are missing in the destination */ + pStmt = db_prepare( + "SELECT name FROM main.sqlite_schema" + " WHERE type='index' AND tbl_name=%Q" + " AND sql IS NOT NULL" + " AND sql NOT IN (SELECT sql FROM aux.sqlite_schema" + " WHERE type='index' AND tbl_name=%Q" + " AND sql IS NOT NULL)", + zTab, zTab); + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + char *z = safeId((const char*)sqlite3_column_text(pStmt,0)); + fprintf(out, "DROP INDEX %s;\n", z); + sqlite3_free(z); + } + sqlite3_finalize(pStmt); + + /* Run the query and output differences */ + if( !g.bSchemaOnly ){ + pStmt = db_prepare("%s", sqlite3_str_value(pSql)); + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + int iType = sqlite3_column_int(pStmt, nPk); + if( iType==1 || iType==2 ){ + if( iType==1 ){ /* Change the content of a row */ + fprintf(out, "%sUPDATE %s", zLead, zId); + zSep = " SET"; + for(i=nPk+1; i<nQ; i+=2){ + if( sqlite3_column_int(pStmt,i)==0 ) continue; + fprintf(out, "%s %s=", zSep, az2[(i+nPk-1)/2]); + zSep = ","; + printQuoted(out, sqlite3_column_value(pStmt,i+1)); + } + }else{ /* Delete a row */ + fprintf(out, "%sDELETE FROM %s", zLead, zId); + } + zSep = " WHERE"; + for(i=0; i<nPk; i++){ + fprintf(out, "%s %s=", zSep, az2[i]); + printQuoted(out, sqlite3_column_value(pStmt,i)); + zSep = " AND"; + } + fprintf(out, ";\n"); + }else{ /* Insert a row */ + fprintf(out, "%sINSERT INTO %s(%s", zLead, zId, az2[0]); + for(i=1; az2[i]; i++) fprintf(out, ",%s", az2[i]); + fprintf(out, ") VALUES"); + zSep = "("; + for(i=0; i<nPk2; i++){ + fprintf(out, "%s", zSep); + zSep = ","; + printQuoted(out, sqlite3_column_value(pStmt,i)); + } + for(i=nPk2+2; i<nQ; i+=2){ + fprintf(out, ","); + printQuoted(out, sqlite3_column_value(pStmt,i)); + } + fprintf(out, ");\n"); + } + } + sqlite3_finalize(pStmt); + } /* endif !g.bSchemaOnly */ + + /* Create indexes that are missing in the source */ + pStmt = db_prepare( + "SELECT sql FROM aux.sqlite_schema" + " WHERE type='index' AND tbl_name=%Q" + " AND sql IS NOT NULL" + " AND sql NOT IN (SELECT sql FROM main.sqlite_schema" + " WHERE type='index' AND tbl_name=%Q" + " AND sql IS NOT NULL)", + zTab, zTab); + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + fprintf(out, "%s;\n", sqlite3_column_text(pStmt,0)); + } + sqlite3_finalize(pStmt); + +end_diff_one_table: + strFree(pSql); + sqlite3_free(zId); + namelistFree(az); + namelistFree(az2); + return; +} + +/* +** Check that table zTab exists and has the same schema in both the "main" +** and "aux" databases currently opened by the global db handle. If they +** do not, output an error message on stderr and exit(1). Otherwise, if +** the schemas do match, return control to the caller. +*/ +static void checkSchemasMatch(const char *zTab){ + sqlite3_stmt *pStmt = db_prepare( + "SELECT A.sql=B.sql FROM main.sqlite_schema A, aux.sqlite_schema B" + " WHERE A.name=%Q AND B.name=%Q", zTab, zTab + ); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( sqlite3_column_int(pStmt,0)==0 ){ + runtimeError("schema changes for table %s", safeId(zTab)); + } + }else{ + runtimeError("table %s missing from one or both databases", safeId(zTab)); + } + sqlite3_finalize(pStmt); +} + +/************************************************************************** +** The following code is copied from fossil. It is used to generate the +** fossil delta blobs sometimes used in RBU update records. +*/ + +typedef unsigned short u16; +typedef unsigned int u32; +typedef unsigned char u8; + +/* +** The width of a hash window in bytes. The algorithm only works if this +** is a power of 2. +*/ +#define NHASH 16 + +/* +** The current state of the rolling hash. +** +** z[] holds the values that have been hashed. z[] is a circular buffer. +** z[i] is the first entry and z[(i+NHASH-1)%NHASH] is the last entry of +** the window. +** +** Hash.a is the sum of all elements of hash.z[]. Hash.b is a weighted +** sum. Hash.b is z[i]*NHASH + z[i+1]*(NHASH-1) + ... + z[i+NHASH-1]*1. +** (Each index for z[] should be module NHASH, of course. The %NHASH operator +** is omitted in the prior expression for brevity.) +*/ +typedef struct hash hash; +struct hash { + u16 a, b; /* Hash values */ + u16 i; /* Start of the hash window */ + char z[NHASH]; /* The values that have been hashed */ +}; + +/* +** Initialize the rolling hash using the first NHASH characters of z[] +*/ +static void hash_init(hash *pHash, const char *z){ + u16 a, b, i; + a = b = 0; + for(i=0; i<NHASH; i++){ + a += z[i]; + b += (NHASH-i)*z[i]; + pHash->z[i] = z[i]; + } + pHash->a = a & 0xffff; + pHash->b = b & 0xffff; + pHash->i = 0; +} + +/* +** Advance the rolling hash by a single character "c" +*/ +static void hash_next(hash *pHash, int c){ + u16 old = pHash->z[pHash->i]; + pHash->z[pHash->i] = (char)c; + pHash->i = (pHash->i+1)&(NHASH-1); + pHash->a = pHash->a - old + (char)c; + pHash->b = pHash->b - NHASH*old + pHash->a; +} + +/* +** Return a 32-bit hash value +*/ +static u32 hash_32bit(hash *pHash){ + return (pHash->a & 0xffff) | (((u32)(pHash->b & 0xffff))<<16); +} + +/* +** Write an base-64 integer into the given buffer. +*/ +static void putInt(unsigned int v, char **pz){ + static const char zDigits[] = + "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz~"; + /* 123456789 123456789 123456789 123456789 123456789 123456789 123 */ + int i, j; + char zBuf[20]; + if( v==0 ){ + *(*pz)++ = '0'; + return; + } + for(i=0; v>0; i++, v>>=6){ + zBuf[i] = zDigits[v&0x3f]; + } + for(j=i-1; j>=0; j--){ + *(*pz)++ = zBuf[j]; + } +} + +/* +** Return the number digits in the base-64 representation of a positive integer +*/ +static int digit_count(int v){ + unsigned int i, x; + for(i=1, x=64; (unsigned int)v>=x; i++, x <<= 6){} + return i; +} + +/* +** Compute a 32-bit checksum on the N-byte buffer. Return the result. +*/ +static unsigned int checksum(const char *zIn, size_t N){ + const unsigned char *z = (const unsigned char *)zIn; + unsigned sum0 = 0; + unsigned sum1 = 0; + unsigned sum2 = 0; + unsigned sum3 = 0; + while(N >= 16){ + sum0 += ((unsigned)z[0] + z[4] + z[8] + z[12]); + sum1 += ((unsigned)z[1] + z[5] + z[9] + z[13]); + sum2 += ((unsigned)z[2] + z[6] + z[10]+ z[14]); + sum3 += ((unsigned)z[3] + z[7] + z[11]+ z[15]); + z += 16; + N -= 16; + } + while(N >= 4){ + sum0 += z[0]; + sum1 += z[1]; + sum2 += z[2]; + sum3 += z[3]; + z += 4; + N -= 4; + } + sum3 += (sum2 << 8) + (sum1 << 16) + (sum0 << 24); + switch(N){ + case 3: sum3 += (z[2] << 8); + case 2: sum3 += (z[1] << 16); + case 1: sum3 += (z[0] << 24); + default: ; + } + return sum3; +} + +/* +** Create a new delta. +** +** The delta is written into a preallocated buffer, zDelta, which +** should be at least 60 bytes longer than the target file, zOut. +** The delta string will be NUL-terminated, but it might also contain +** embedded NUL characters if either the zSrc or zOut files are +** binary. This function returns the length of the delta string +** in bytes, excluding the final NUL terminator character. +** +** Output Format: +** +** The delta begins with a base64 number followed by a newline. This +** number is the number of bytes in the TARGET file. Thus, given a +** delta file z, a program can compute the size of the output file +** simply by reading the first line and decoding the base-64 number +** found there. The delta_output_size() routine does exactly this. +** +** After the initial size number, the delta consists of a series of +** literal text segments and commands to copy from the SOURCE file. +** A copy command looks like this: +** +** NNN@MMM, +** +** where NNN is the number of bytes to be copied and MMM is the offset +** into the source file of the first byte (both base-64). If NNN is 0 +** it means copy the rest of the input file. Literal text is like this: +** +** NNN:TTTTT +** +** where NNN is the number of bytes of text (base-64) and TTTTT is the text. +** +** The last term is of the form +** +** NNN; +** +** In this case, NNN is a 32-bit bigendian checksum of the output file +** that can be used to verify that the delta applied correctly. All +** numbers are in base-64. +** +** Pure text files generate a pure text delta. Binary files generate a +** delta that may contain some binary data. +** +** Algorithm: +** +** The encoder first builds a hash table to help it find matching +** patterns in the source file. 16-byte chunks of the source file +** sampled at evenly spaced intervals are used to populate the hash +** table. +** +** Next we begin scanning the target file using a sliding 16-byte +** window. The hash of the 16-byte window in the target is used to +** search for a matching section in the source file. When a match +** is found, a copy command is added to the delta. An effort is +** made to extend the matching section to regions that come before +** and after the 16-byte hash window. A copy command is only issued +** if the result would use less space that just quoting the text +** literally. Literal text is added to the delta for sections that +** do not match or which can not be encoded efficiently using copy +** commands. +*/ +static int rbuDeltaCreate( + const char *zSrc, /* The source or pattern file */ + unsigned int lenSrc, /* Length of the source file */ + const char *zOut, /* The target file */ + unsigned int lenOut, /* Length of the target file */ + char *zDelta /* Write the delta into this buffer */ +){ + unsigned int i, base; + char *zOrigDelta = zDelta; + hash h; + int nHash; /* Number of hash table entries */ + int *landmark; /* Primary hash table */ + int *collide; /* Collision chain */ + int lastRead = -1; /* Last byte of zSrc read by a COPY command */ + + /* Add the target file size to the beginning of the delta + */ + putInt(lenOut, &zDelta); + *(zDelta++) = '\n'; + + /* If the source file is very small, it means that we have no + ** chance of ever doing a copy command. Just output a single + ** literal segment for the entire target and exit. + */ + if( lenSrc<=NHASH ){ + putInt(lenOut, &zDelta); + *(zDelta++) = ':'; + memcpy(zDelta, zOut, lenOut); + zDelta += lenOut; + putInt(checksum(zOut, lenOut), &zDelta); + *(zDelta++) = ';'; + return (int)(zDelta - zOrigDelta); + } + + /* Compute the hash table used to locate matching sections in the + ** source file. + */ + nHash = lenSrc/NHASH; + collide = sqlite3_malloc( nHash*2*sizeof(int) ); + landmark = &collide[nHash]; + memset(landmark, -1, nHash*sizeof(int)); + memset(collide, -1, nHash*sizeof(int)); + for(i=0; i<lenSrc-NHASH; i+=NHASH){ + int hv; + hash_init(&h, &zSrc[i]); + hv = hash_32bit(&h) % nHash; + collide[i/NHASH] = landmark[hv]; + landmark[hv] = i/NHASH; + } + + /* Begin scanning the target file and generating copy commands and + ** literal sections of the delta. + */ + base = 0; /* We have already generated everything before zOut[base] */ + while( base+NHASH<lenOut ){ + int iSrc, iBlock; + int bestCnt, bestOfst=0, bestLitsz=0; + hash_init(&h, &zOut[base]); + i = 0; /* Trying to match a landmark against zOut[base+i] */ + bestCnt = 0; + while( 1 ){ + int hv; + int limit = 250; + + hv = hash_32bit(&h) % nHash; + iBlock = landmark[hv]; + while( iBlock>=0 && (limit--)>0 ){ + /* + ** The hash window has identified a potential match against + ** landmark block iBlock. But we need to investigate further. + ** + ** Look for a region in zOut that matches zSrc. Anchor the search + ** at zSrc[iSrc] and zOut[base+i]. Do not include anything prior to + ** zOut[base] or after zOut[outLen] nor anything after zSrc[srcLen]. + ** + ** Set cnt equal to the length of the match and set ofst so that + ** zSrc[ofst] is the first element of the match. litsz is the number + ** of characters between zOut[base] and the beginning of the match. + ** sz will be the overhead (in bytes) needed to encode the copy + ** command. Only generate copy command if the overhead of the + ** copy command is less than the amount of literal text to be copied. + */ + int cnt, ofst, litsz; + int j, k, x, y; + int sz; + + /* Beginning at iSrc, match forwards as far as we can. j counts + ** the number of characters that match */ + iSrc = iBlock*NHASH; + for( + j=0, x=iSrc, y=base+i; + (unsigned int)x<lenSrc && (unsigned int)y<lenOut; + j++, x++, y++ + ){ + if( zSrc[x]!=zOut[y] ) break; + } + j--; + + /* Beginning at iSrc-1, match backwards as far as we can. k counts + ** the number of characters that match */ + for(k=1; k<iSrc && (unsigned int)k<=i; k++){ + if( zSrc[iSrc-k]!=zOut[base+i-k] ) break; + } + k--; + + /* Compute the offset and size of the matching region */ + ofst = iSrc-k; + cnt = j+k+1; + litsz = i-k; /* Number of bytes of literal text before the copy */ + /* sz will hold the number of bytes needed to encode the "insert" + ** command and the copy command, not counting the "insert" text */ + sz = digit_count(i-k)+digit_count(cnt)+digit_count(ofst)+3; + if( cnt>=sz && cnt>bestCnt ){ + /* Remember this match only if it is the best so far and it + ** does not increase the file size */ + bestCnt = cnt; + bestOfst = iSrc-k; + bestLitsz = litsz; + } + + /* Check the next matching block */ + iBlock = collide[iBlock]; + } + + /* We have a copy command that does not cause the delta to be larger + ** than a literal insert. So add the copy command to the delta. + */ + if( bestCnt>0 ){ + if( bestLitsz>0 ){ + /* Add an insert command before the copy */ + putInt(bestLitsz,&zDelta); + *(zDelta++) = ':'; + memcpy(zDelta, &zOut[base], bestLitsz); + zDelta += bestLitsz; + base += bestLitsz; + } + base += bestCnt; + putInt(bestCnt, &zDelta); + *(zDelta++) = '@'; + putInt(bestOfst, &zDelta); + *(zDelta++) = ','; + if( bestOfst + bestCnt -1 > lastRead ){ + lastRead = bestOfst + bestCnt - 1; + } + bestCnt = 0; + break; + } + + /* If we reach this point, it means no match is found so far */ + if( base+i+NHASH>=lenOut ){ + /* We have reached the end of the file and have not found any + ** matches. Do an "insert" for everything that does not match */ + putInt(lenOut-base, &zDelta); + *(zDelta++) = ':'; + memcpy(zDelta, &zOut[base], lenOut-base); + zDelta += lenOut-base; + base = lenOut; + break; + } + + /* Advance the hash by one character. Keep looking for a match */ + hash_next(&h, zOut[base+i+NHASH]); + i++; + } + } + /* Output a final "insert" record to get all the text at the end of + ** the file that does not match anything in the source file. + */ + if( base<lenOut ){ + putInt(lenOut-base, &zDelta); + *(zDelta++) = ':'; + memcpy(zDelta, &zOut[base], lenOut-base); + zDelta += lenOut-base; + } + /* Output the final checksum record. */ + putInt(checksum(zOut, lenOut), &zDelta); + *(zDelta++) = ';'; + sqlite3_free(collide); + return (int)(zDelta - zOrigDelta); +} + +/* +** End of code copied from fossil. +**************************************************************************/ + +static void strPrintfArray( + sqlite3_str *pStr, /* String object to append to */ + const char *zSep, /* Separator string */ + const char *zFmt, /* Format for each entry */ + char **az, int n /* Array of strings & its size (or -1) */ +){ + int i; + for(i=0; az[i] && (i<n || n<0); i++){ + if( i!=0 ) sqlite3_str_appendf(pStr, "%s", zSep); + sqlite3_str_appendf(pStr, zFmt, az[i], az[i], az[i]); + } +} + +static void getRbudiffQuery( + const char *zTab, + char **azCol, + int nPK, + int bOtaRowid, + sqlite3_str *pSql +){ + int i; + + /* First the newly inserted rows: **/ + sqlite3_str_appendf(pSql, "SELECT "); + strPrintfArray(pSql, ", ", "%s", azCol, -1); + sqlite3_str_appendf(pSql, ", 0, "); /* Set ota_control to 0 for an insert */ + strPrintfArray(pSql, ", ", "NULL", azCol, -1); + sqlite3_str_appendf(pSql, " FROM aux.%Q AS n WHERE NOT EXISTS (\n", zTab); + sqlite3_str_appendf(pSql, " SELECT 1 FROM ", zTab); + sqlite3_str_appendf(pSql, " main.%Q AS o WHERE ", zTab); + strPrintfArray(pSql, " AND ", "(n.%Q = o.%Q)", azCol, nPK); + sqlite3_str_appendf(pSql, "\n) AND "); + strPrintfArray(pSql, " AND ", "(n.%Q IS NOT NULL)", azCol, nPK); + + /* Deleted rows: */ + sqlite3_str_appendf(pSql, "\nUNION ALL\nSELECT "); + strPrintfArray(pSql, ", ", "%s", azCol, nPK); + if( azCol[nPK] ){ + sqlite3_str_appendf(pSql, ", "); + strPrintfArray(pSql, ", ", "NULL", &azCol[nPK], -1); + } + sqlite3_str_appendf(pSql, ", 1, "); /* Set ota_control to 1 for a delete */ + strPrintfArray(pSql, ", ", "NULL", azCol, -1); + sqlite3_str_appendf(pSql, " FROM main.%Q AS n WHERE NOT EXISTS (\n", zTab); + sqlite3_str_appendf(pSql, " SELECT 1 FROM ", zTab); + sqlite3_str_appendf(pSql, " aux.%Q AS o WHERE ", zTab); + strPrintfArray(pSql, " AND ", "(n.%Q = o.%Q)", azCol, nPK); + sqlite3_str_appendf(pSql, "\n) AND "); + strPrintfArray(pSql, " AND ", "(n.%Q IS NOT NULL)", azCol, nPK); + + /* Updated rows. If all table columns are part of the primary key, there + ** can be no updates. In this case this part of the compound SELECT can + ** be omitted altogether. */ + if( azCol[nPK] ){ + sqlite3_str_appendf(pSql, "\nUNION ALL\nSELECT "); + strPrintfArray(pSql, ", ", "n.%s", azCol, nPK); + sqlite3_str_appendf(pSql, ",\n"); + strPrintfArray(pSql, " ,\n", + " CASE WHEN n.%s IS o.%s THEN NULL ELSE n.%s END", &azCol[nPK], -1 + ); + + if( bOtaRowid==0 ){ + sqlite3_str_appendf(pSql, ", '"); + strPrintfArray(pSql, "", ".", azCol, nPK); + sqlite3_str_appendf(pSql, "' ||\n"); + }else{ + sqlite3_str_appendf(pSql, ",\n"); + } + strPrintfArray(pSql, " ||\n", + " CASE WHEN n.%s IS o.%s THEN '.' ELSE 'x' END", &azCol[nPK], -1 + ); + sqlite3_str_appendf(pSql, "\nAS ota_control, "); + strPrintfArray(pSql, ", ", "NULL", azCol, nPK); + sqlite3_str_appendf(pSql, ",\n"); + strPrintfArray(pSql, " ,\n", + " CASE WHEN n.%s IS o.%s THEN NULL ELSE o.%s END", &azCol[nPK], -1 + ); + + sqlite3_str_appendf(pSql, "\nFROM main.%Q AS o, aux.%Q AS n\nWHERE ", + zTab, zTab); + strPrintfArray(pSql, " AND ", "(n.%Q = o.%Q)", azCol, nPK); + sqlite3_str_appendf(pSql, " AND ota_control LIKE '%%x%%'"); + } + + /* Now add an ORDER BY clause to sort everything by PK. */ + sqlite3_str_appendf(pSql, "\nORDER BY "); + for(i=1; i<=nPK; i++) sqlite3_str_appendf(pSql, "%s%d", ((i>1)?", ":""), i); +} + +static void rbudiff_one_table(const char *zTab, FILE *out){ + int bOtaRowid; /* True to use an ota_rowid column */ + int nPK; /* Number of primary key columns in table */ + char **azCol; /* NULL terminated array of col names */ + int i; + int nCol; + sqlite3_str *pCt; /* The "CREATE TABLE data_xxx" statement */ + sqlite3_str *pSql; /* Query to find differences */ + sqlite3_str *pInsert; /* First part of output INSERT statement */ + sqlite3_stmt *pStmt = 0; + int nRow = 0; /* Total rows in data_xxx table */ + + /* --rbu mode must use real primary keys. */ + g.bSchemaPK = 1; + pCt = sqlite3_str_new(0); + pSql = sqlite3_str_new(0); + pInsert = sqlite3_str_new(0); + + /* Check that the schemas of the two tables match. Exit early otherwise. */ + checkSchemasMatch(zTab); + + /* Grab the column names and PK details for the table(s). If no usable PK + ** columns are found, bail out early. */ + azCol = columnNames("main", zTab, &nPK, &bOtaRowid); + if( azCol==0 ){ + runtimeError("table %s has no usable PK columns", zTab); + } + for(nCol=0; azCol[nCol]; nCol++); + + /* Build and output the CREATE TABLE statement for the data_xxx table */ + sqlite3_str_appendf(pCt, "CREATE TABLE IF NOT EXISTS 'data_%q'(", zTab); + if( bOtaRowid ) sqlite3_str_appendf(pCt, "rbu_rowid, "); + strPrintfArray(pCt, ", ", "%s", &azCol[bOtaRowid], -1); + sqlite3_str_appendf(pCt, ", rbu_control);"); + + /* Get the SQL for the query to retrieve data from the two databases */ + getRbudiffQuery(zTab, azCol, nPK, bOtaRowid, pSql); + + /* Build the first part of the INSERT statement output for each row + ** in the data_xxx table. */ + sqlite3_str_appendf(pInsert, "INSERT INTO 'data_%q' (", zTab); + if( bOtaRowid ) sqlite3_str_appendf(pInsert, "rbu_rowid, "); + strPrintfArray(pInsert, ", ", "%s", &azCol[bOtaRowid], -1); + sqlite3_str_appendf(pInsert, ", rbu_control) VALUES("); + + pStmt = db_prepare("%s", sqlite3_str_value(pSql)); + + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + + /* If this is the first row output, print out the CREATE TABLE + ** statement first. And reset pCt so that it will not be + ** printed again. */ + if( sqlite3_str_length(pCt) ){ + fprintf(out, "%s\n", sqlite3_str_value(pCt)); + sqlite3_str_reset(pCt); + } + + /* Output the first part of the INSERT statement */ + fprintf(out, "%s", sqlite3_str_value(pInsert)); + nRow++; + + if( sqlite3_column_type(pStmt, nCol)==SQLITE_INTEGER ){ + for(i=0; i<=nCol; i++){ + if( i>0 ) fprintf(out, ", "); + printQuoted(out, sqlite3_column_value(pStmt, i)); + } + }else{ + char *zOtaControl; + int nOtaControl = sqlite3_column_bytes(pStmt, nCol); + + zOtaControl = (char*)sqlite3_malloc(nOtaControl+1); + memcpy(zOtaControl, sqlite3_column_text(pStmt, nCol), nOtaControl+1); + + for(i=0; i<nCol; i++){ + int bDone = 0; + if( i>=nPK + && sqlite3_column_type(pStmt, i)==SQLITE_BLOB + && sqlite3_column_type(pStmt, nCol+1+i)==SQLITE_BLOB + ){ + const char *aSrc = sqlite3_column_blob(pStmt, nCol+1+i); + int nSrc = sqlite3_column_bytes(pStmt, nCol+1+i); + const char *aFinal = sqlite3_column_blob(pStmt, i); + int nFinal = sqlite3_column_bytes(pStmt, i); + char *aDelta; + int nDelta; + + aDelta = sqlite3_malloc(nFinal + 60); + nDelta = rbuDeltaCreate(aSrc, nSrc, aFinal, nFinal, aDelta); + if( nDelta<nFinal ){ + int j; + fprintf(out, "x'"); + for(j=0; j<nDelta; j++) fprintf(out, "%02x", (u8)aDelta[j]); + fprintf(out, "'"); + zOtaControl[i-bOtaRowid] = 'f'; + bDone = 1; + } + sqlite3_free(aDelta); + } + + if( bDone==0 ){ + printQuoted(out, sqlite3_column_value(pStmt, i)); + } + fprintf(out, ", "); + } + fprintf(out, "'%s'", zOtaControl); + sqlite3_free(zOtaControl); + } + + /* And the closing bracket of the insert statement */ + fprintf(out, ");\n"); + } + + sqlite3_finalize(pStmt); + if( nRow>0 ){ + sqlite3_str *pCnt = sqlite3_str_new(0); + sqlite3_str_appendf(pCnt, + "INSERT INTO rbu_count VALUES('data_%q', %d);", zTab, nRow); + fprintf(out, "%s\n", sqlite3_str_value(pCnt)); + strFree(pCnt); + } + + strFree(pCt); + strFree(pSql); + strFree(pInsert); +} + +/* +** Display a summary of differences between two versions of the same +** table table. +** +** * Number of rows changed +** * Number of rows added +** * Number of rows deleted +** * Number of identical rows +*/ +static void summarize_one_table(const char *zTab, FILE *out){ + char *zId = safeId(zTab); /* Name of table (translated for us in SQL) */ + char **az = 0; /* Columns in main */ + char **az2 = 0; /* Columns in aux */ + int nPk; /* Primary key columns in main */ + int nPk2; /* Primary key columns in aux */ + int n = 0; /* Number of columns in main */ + int n2; /* Number of columns in aux */ + int i; /* Loop counter */ + const char *zSep; /* Separator string */ + sqlite3_str *pSql; /* Comparison query */ + sqlite3_stmt *pStmt; /* Query statement to do the diff */ + sqlite3_int64 nUpdate; /* Number of updated rows */ + sqlite3_int64 nUnchanged; /* Number of unmodified rows */ + sqlite3_int64 nDelete; /* Number of deleted rows */ + sqlite3_int64 nInsert; /* Number of inserted rows */ + + pSql = sqlite3_str_new(0); + if( sqlite3_table_column_metadata(g.db,"aux",zTab,0,0,0,0,0,0) ){ + if( !sqlite3_table_column_metadata(g.db,"main",zTab,0,0,0,0,0,0) ){ + /* Table missing from second database. */ + Wfprintf(out, "%s: missing from second database\n", zTab); + } + goto end_summarize_one_table; + } + + if( sqlite3_table_column_metadata(g.db,"main",zTab,0,0,0,0,0,0) ){ + /* Table missing from source */ + Wfprintf(out, "%s: missing from first database\n", zTab); + goto end_summarize_one_table; + } + + az = columnNames("main", zTab, &nPk, 0); + az2 = columnNames("aux", zTab, &nPk2, 0); + if( az && az2 ){ + for(n=0; az[n]; n++){ + if( sqlite3_stricmp(az[n],az2[n])!=0 ) break; + } + } + if( az==0 + || az2==0 + || nPk!=nPk2 + || az[n] + ){ + /* Schema mismatch */ + Wfprintf(out, "%s: incompatible schema\n", zTab); + goto end_summarize_one_table; + } + + /* Build the comparison query */ + for(n2=n; az[n2]; n2++){} + sqlite3_str_appendf(pSql, "SELECT 1, count(*)"); + if( n2==nPk2 ){ + sqlite3_str_appendf(pSql, ", 0\n"); + }else{ + zSep = ", sum("; + for(i=nPk; az[i]; i++){ + sqlite3_str_appendf(pSql, "%sA.%s IS NOT B.%s", zSep, az[i], az[i]); + zSep = " OR "; + } + sqlite3_str_appendf(pSql, ")\n"); + } + sqlite3_str_appendf(pSql, " FROM main.%s A, aux.%s B\n", zId, zId); + zSep = " WHERE"; + for(i=0; i<nPk; i++){ + sqlite3_str_appendf(pSql, "%s A.%s=B.%s", zSep, az[i], az[i]); + zSep = " AND"; + } + sqlite3_str_appendf(pSql, " UNION ALL\n"); + sqlite3_str_appendf(pSql, "SELECT 2, count(*), 0\n"); + sqlite3_str_appendf(pSql, " FROM main.%s A\n", zId); + sqlite3_str_appendf(pSql, " WHERE NOT EXISTS(SELECT 1 FROM aux.%s B ", zId); + zSep = "WHERE"; + for(i=0; i<nPk; i++){ + sqlite3_str_appendf(pSql, "%s A.%s=B.%s", zSep, az[i], az[i]); + zSep = " AND"; + } + sqlite3_str_appendf(pSql, ")\n"); + sqlite3_str_appendf(pSql, " UNION ALL\n"); + sqlite3_str_appendf(pSql, "SELECT 3, count(*), 0\n"); + sqlite3_str_appendf(pSql, " FROM aux.%s B\n", zId); + sqlite3_str_appendf(pSql, " WHERE NOT EXISTS(SELECT 1 FROM main.%s A ", zId); + zSep = "WHERE"; + for(i=0; i<nPk; i++){ + sqlite3_str_appendf(pSql, "%s A.%s=B.%s", zSep, az[i], az[i]); + zSep = " AND"; + } + sqlite3_str_appendf(pSql, ")\n ORDER BY 1;\n"); + + if( (g.fDebug & DEBUG_DIFF_SQL)!=0 ){ + Wfprintf(stdout, "SQL for %s:\n%s\n", zId, sqlite3_str_value(pSql)); + goto end_summarize_one_table; + } + + /* Run the query and output difference summary */ + pStmt = db_prepare("%s", sqlite3_str_value(pSql)); + nUpdate = 0; + nInsert = 0; + nDelete = 0; + nUnchanged = 0; + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + switch( sqlite3_column_int(pStmt,0) ){ + case 1: + nUpdate = sqlite3_column_int64(pStmt,2); + nUnchanged = sqlite3_column_int64(pStmt,1) - nUpdate; + break; + case 2: + nDelete = sqlite3_column_int64(pStmt,1); + break; + case 3: + nInsert = sqlite3_column_int64(pStmt,1); + break; + } + } + sqlite3_finalize(pStmt); + Wfprintf(out, + "%s: %lld changes, %lld inserts, %lld deletes, %lld unchanged\n", + zTab, nUpdate, nInsert, nDelete, nUnchanged); + +end_summarize_one_table: + strFree(pSql); + sqlite3_free(zId); + namelistFree(az); + namelistFree(az2); + return; +} + +/* +** Write a 64-bit signed integer as a varint onto out +*/ +static void putsVarint(FILE *out, sqlite3_uint64 v){ + int i, n; + unsigned char p[12]; + if( v & (((sqlite3_uint64)0xff000000)<<32) ){ + p[8] = (unsigned char)v; + v >>= 8; + for(i=7; i>=0; i--){ + p[i] = (unsigned char)((v & 0x7f) | 0x80); + v >>= 7; + } + fwrite(p, 8, 1, out); + }else{ + n = 9; + do{ + p[n--] = (unsigned char)((v & 0x7f) | 0x80); + v >>= 7; + }while( v!=0 ); + p[9] &= 0x7f; + fwrite(p+n+1, 9-n, 1, out); + } +} + +/* +** Write an SQLite value onto out. +*/ +static void putValue(FILE *out, sqlite3_stmt *pStmt, int k){ + int iDType = sqlite3_column_type(pStmt, k); + sqlite3_int64 iX; + double rX; + sqlite3_uint64 uX; + int j; + + putc(iDType, out); + switch( iDType ){ + case SQLITE_INTEGER: + iX = sqlite3_column_int64(pStmt, k); + memcpy(&uX, &iX, 8); + for(j=56; j>=0; j-=8) putc((uX>>j)&0xff, out); + break; + case SQLITE_FLOAT: + rX = sqlite3_column_double(pStmt, k); + memcpy(&uX, &rX, 8); + for(j=56; j>=0; j-=8) putc((uX>>j)&0xff, out); + break; + case SQLITE_TEXT: + iX = sqlite3_column_bytes(pStmt, k); + putsVarint(out, (sqlite3_uint64)iX); + fwrite(sqlite3_column_text(pStmt, k),1,(size_t)iX,out); + break; + case SQLITE_BLOB: + iX = sqlite3_column_bytes(pStmt, k); + putsVarint(out, (sqlite3_uint64)iX); + fwrite(sqlite3_column_blob(pStmt, k),1,(size_t)iX,out); + break; + case SQLITE_NULL: + break; + } +} + +/* +** Generate a CHANGESET for all differences from main.zTab to aux.zTab. +*/ +static void changeset_one_table(const char *zTab, FILE *out){ + sqlite3_stmt *pStmt; /* SQL statment */ + char *zId = safeId(zTab); /* Escaped name of the table */ + char **azCol = 0; /* List of escaped column names */ + int nCol = 0; /* Number of columns */ + int *aiFlg = 0; /* 0 if column is not part of PK */ + int *aiPk = 0; /* Column numbers for each PK column */ + int nPk = 0; /* Number of PRIMARY KEY columns */ + sqlite3_str *pSql; /* SQL for the diff query */ + int i, k; /* Loop counters */ + const char *zSep; /* List separator */ + + /* Check that the schemas of the two tables match. Exit early otherwise. */ + checkSchemasMatch(zTab); + pSql = sqlite3_str_new(0); + + pStmt = db_prepare("PRAGMA main.table_info=%Q", zTab); + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + nCol++; + azCol = sqlite3_realloc(azCol, sizeof(char*)*nCol); + if( azCol==0 ) runtimeError("out of memory"); + aiFlg = sqlite3_realloc(aiFlg, sizeof(int)*nCol); + if( aiFlg==0 ) runtimeError("out of memory"); + azCol[nCol-1] = safeId((const char*)sqlite3_column_text(pStmt,1)); + aiFlg[nCol-1] = i = sqlite3_column_int(pStmt,5); + if( i>0 ){ + if( i>nPk ){ + nPk = i; + aiPk = sqlite3_realloc(aiPk, sizeof(int)*nPk); + if( aiPk==0 ) runtimeError("out of memory"); + } + aiPk[i-1] = nCol-1; + } + } + sqlite3_finalize(pStmt); + if( nPk==0 ) goto end_changeset_one_table; + if( nCol>nPk ){ + sqlite3_str_appendf(pSql, "SELECT %d", SQLITE_UPDATE); + for(i=0; i<nCol; i++){ + if( aiFlg[i] ){ + sqlite3_str_appendf(pSql, ",\n A.%s", azCol[i]); + }else{ + sqlite3_str_appendf(pSql, ",\n A.%s IS NOT B.%s, A.%s, B.%s", + azCol[i], azCol[i], azCol[i], azCol[i]); + } + } + sqlite3_str_appendf(pSql,"\n FROM main.%s A, aux.%s B\n", zId, zId); + zSep = " WHERE"; + for(i=0; i<nPk; i++){ + sqlite3_str_appendf(pSql, "%s A.%s=B.%s", + zSep, azCol[aiPk[i]], azCol[aiPk[i]]); + zSep = " AND"; + } + zSep = "\n AND ("; + for(i=0; i<nCol; i++){ + if( aiFlg[i] ) continue; + sqlite3_str_appendf(pSql, "%sA.%s IS NOT B.%s", zSep, azCol[i], azCol[i]); + zSep = " OR\n "; + } + sqlite3_str_appendf(pSql,")\n UNION ALL\n"); + } + sqlite3_str_appendf(pSql, "SELECT %d", SQLITE_DELETE); + for(i=0; i<nCol; i++){ + if( aiFlg[i] ){ + sqlite3_str_appendf(pSql, ",\n A.%s", azCol[i]); + }else{ + sqlite3_str_appendf(pSql, ",\n 1, A.%s, NULL", azCol[i]); + } + } + sqlite3_str_appendf(pSql, "\n FROM main.%s A\n", zId); + sqlite3_str_appendf(pSql, " WHERE NOT EXISTS(SELECT 1 FROM aux.%s B\n", zId); + zSep = " WHERE"; + for(i=0; i<nPk; i++){ + sqlite3_str_appendf(pSql, "%s A.%s=B.%s", + zSep, azCol[aiPk[i]], azCol[aiPk[i]]); + zSep = " AND"; + } + sqlite3_str_appendf(pSql, ")\n UNION ALL\n"); + sqlite3_str_appendf(pSql, "SELECT %d", SQLITE_INSERT); + for(i=0; i<nCol; i++){ + if( aiFlg[i] ){ + sqlite3_str_appendf(pSql, ",\n B.%s", azCol[i]); + }else{ + sqlite3_str_appendf(pSql, ",\n 1, NULL, B.%s", azCol[i]); + } + } + sqlite3_str_appendf(pSql, "\n FROM aux.%s B\n", zId); + sqlite3_str_appendf(pSql, " WHERE NOT EXISTS(SELECT 1 FROM main.%s A\n", zId); + zSep = " WHERE"; + for(i=0; i<nPk; i++){ + sqlite3_str_appendf(pSql, "%s A.%s=B.%s", + zSep, azCol[aiPk[i]], azCol[aiPk[i]]); + zSep = " AND"; + } + sqlite3_str_appendf(pSql, ")\n"); + sqlite3_str_appendf(pSql, " ORDER BY"); + zSep = " "; + for(i=0; i<nPk; i++){ + sqlite3_str_appendf(pSql, "%s %d", zSep, aiPk[i]+2); + zSep = ","; + } + sqlite3_str_appendf(pSql, ";\n"); + + if( g.fDebug & DEBUG_DIFF_SQL ){ + Wfprintf(stdout, "SQL for %s:\n%s\n", zId, sqlite3_str_value(pSql)); + goto end_changeset_one_table; + } + + putc('T', out); + putsVarint(out, (sqlite3_uint64)nCol); + for(i=0; i<nCol; i++) putc(aiFlg[i], out); + fwrite(zTab, 1, strlen(zTab), out); + putc(0, out); + + pStmt = db_prepare("%s", sqlite3_str_value(pSql)); + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + int iType = sqlite3_column_int(pStmt,0); + putc(iType, out); + putc(0, out); + switch( sqlite3_column_int(pStmt,0) ){ + case SQLITE_UPDATE: { + for(k=1, i=0; i<nCol; i++){ + if( aiFlg[i] ){ + putValue(out, pStmt, k); + k++; + }else if( sqlite3_column_int(pStmt,k) ){ + putValue(out, pStmt, k+1); + k += 3; + }else{ + putc(0, out); + k += 3; + } + } + for(k=1, i=0; i<nCol; i++){ + if( aiFlg[i] ){ + putc(0, out); + k++; + }else if( sqlite3_column_int(pStmt,k) ){ + putValue(out, pStmt, k+2); + k += 3; + }else{ + putc(0, out); + k += 3; + } + } + break; + } + case SQLITE_INSERT: { + for(k=1, i=0; i<nCol; i++){ + if( aiFlg[i] ){ + putValue(out, pStmt, k); + k++; + }else{ + putValue(out, pStmt, k+2); + k += 3; + } + } + break; + } + case SQLITE_DELETE: { + for(k=1, i=0; i<nCol; i++){ + if( aiFlg[i] ){ + putValue(out, pStmt, k); + k++; + }else{ + putValue(out, pStmt, k+1); + k += 3; + } + } + break; + } + } + } + sqlite3_finalize(pStmt); + +end_changeset_one_table: + while( nCol>0 ) sqlite3_free(azCol[--nCol]); + sqlite3_free(azCol); + sqlite3_free(aiPk); + sqlite3_free(zId); + sqlite3_free(aiFlg); + strFree(pSql); +} + +/* +** Return true if the ascii character passed as the only argument is a +** whitespace character. Otherwise return false. +*/ +static int is_whitespace(char x){ + return (x==' ' || x=='\t' || x=='\n' || x=='\r'); +} + +/* +** Extract the next SQL keyword or quoted string from buffer zIn and copy it +** (or a prefix of it if it will not fit) into buffer zBuf, size nBuf bytes. +** Return a pointer to the character within zIn immediately following +** the token or quoted string just extracted. +*/ +static const char *gobble_token(const char *zIn, char *zBuf, int nBuf){ + const char *p = zIn; + char *pOut = zBuf; + char *pEnd = &pOut[nBuf-1]; + char q = 0; /* quote character, if any */ + + if( p==0 ) return 0; + while( is_whitespace(*p) ) p++; + switch( *p ){ + case '"': q = '"'; break; + case '\'': q = '\''; break; + case '`': q = '`'; break; + case '[': q = ']'; break; + } + + if( q ){ + p++; + while( *p && pOut<pEnd ){ + if( *p==q ){ + p++; + if( *p!=q ) break; + } + if( pOut<pEnd ) *pOut++ = *p; + p++; + } + }else{ + while( *p && !is_whitespace(*p) && *p!='(' ){ + if( pOut<pEnd ) *pOut++ = *p; + p++; + } + } + + *pOut = '\0'; + return p; +} + +/* +** This function is the implementation of SQL scalar function "module_name": +** +** module_name(SQL) +** +** The only argument should be an SQL statement of the type that may appear +** in the sqlite_schema table. If the statement is a "CREATE VIRTUAL TABLE" +** statement, then the value returned is the name of the module that it +** uses. Otherwise, if the statement is not a CVT, NULL is returned. +*/ +static void module_name_func( + sqlite3_context *pCtx, + int nVal, sqlite3_value **apVal +){ + const char *zSql; + char zToken[32]; + + assert( nVal==1 ); + zSql = (const char*)sqlite3_value_text(apVal[0]); + + zSql = gobble_token(zSql, zToken, sizeof(zToken)); + if( zSql==0 || sqlite3_stricmp(zToken, "create") ) return; + zSql = gobble_token(zSql, zToken, sizeof(zToken)); + if( zSql==0 || sqlite3_stricmp(zToken, "virtual") ) return; + zSql = gobble_token(zSql, zToken, sizeof(zToken)); + if( zSql==0 || sqlite3_stricmp(zToken, "table") ) return; + zSql = gobble_token(zSql, zToken, sizeof(zToken)); + if( zSql==0 ) return; + zSql = gobble_token(zSql, zToken, sizeof(zToken)); + if( zSql==0 || sqlite3_stricmp(zToken, "using") ) return; + zSql = gobble_token(zSql, zToken, sizeof(zToken)); + + sqlite3_result_text(pCtx, zToken, -1, SQLITE_TRANSIENT); +} + +/* +** Return the text of an SQL statement that itself returns the list of +** tables to process within the database. +*/ +const char *all_tables_sql(){ + if( g.bHandleVtab ){ + int rc; + + rc = sqlite3_exec(g.db, + "CREATE TEMP TABLE tblmap(module COLLATE nocase, postfix);" + "INSERT INTO temp.tblmap VALUES" + "('fts3', '_content'), ('fts3', '_segments'), ('fts3', '_segdir')," + + "('fts4', '_content'), ('fts4', '_segments'), ('fts4', '_segdir')," + "('fts4', '_docsize'), ('fts4', '_stat')," + + "('fts5', '_data'), ('fts5', '_idx'), ('fts5', '_content')," + "('fts5', '_docsize'), ('fts5', '_config')," + + "('rtree', '_node'), ('rtree', '_rowid'), ('rtree', '_parent');" + , 0, 0, 0 + ); + assert( rc==SQLITE_OK ); + + rc = sqlite3_create_function( + g.db, "module_name", 1, SQLITE_UTF8, 0, module_name_func, 0, 0 + ); + assert( rc==SQLITE_OK ); + + return + "SELECT name FROM main.sqlite_schema\n" + " WHERE type='table' AND (\n" + " module_name(sql) IS NULL OR \n" + " module_name(sql) IN (SELECT module FROM temp.tblmap)\n" + " ) AND name NOT IN (\n" + " SELECT a.name || b.postfix \n" + "FROM main.sqlite_schema AS a, temp.tblmap AS b \n" + "WHERE module_name(a.sql) = b.module\n" + " )\n" + "UNION \n" + "SELECT name FROM aux.sqlite_schema\n" + " WHERE type='table' AND (\n" + " module_name(sql) IS NULL OR \n" + " module_name(sql) IN (SELECT module FROM temp.tblmap)\n" + " ) AND name NOT IN (\n" + " SELECT a.name || b.postfix \n" + "FROM aux.sqlite_schema AS a, temp.tblmap AS b \n" + "WHERE module_name(a.sql) = b.module\n" + " )\n" + " ORDER BY name"; + }else{ + return + "SELECT name FROM main.sqlite_schema\n" + " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n" + " UNION\n" + "SELECT name FROM aux.sqlite_schema\n" + " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n" + " ORDER BY name"; + } +} + +/* +** Print sketchy documentation for this utility program +*/ +static void showHelp(void){ + Wfprintf(stdout, "Usage: %s [options] DB1 DB2\n", g.zArgv0); + Wfprintf(stdout, +"Output SQL text that would transform DB1 into DB2.\n" +"Options:\n" +" --changeset FILE Write a CHANGESET into FILE\n" +" -L|--lib LIBRARY Load an SQLite extension library\n" +" --primarykey Use schema-defined PRIMARY KEYs\n" +" --rbu Output SQL to create/populate RBU table(s)\n" +" --schema Show only differences in the schema\n" +" --summary Show only a summary of the differences\n" +" --table TAB Show only differences in table TAB\n" +" --transaction Show SQL output inside a transaction\n" +" --vtab Handle fts3, fts4, fts5 and rtree tables\n" +"See https://sqlite.org/sqldiff.html for detailed explanation.\n" + ); +} + +int main(int argc, char **argv){ + const char *zDb1 = 0; + const char *zDb2 = 0; + int i; + int rc; + char *zErrMsg = 0; + char *zSql; + sqlite3_stmt *pStmt; + char *zTab = 0; + FILE *out = stdout; + void (*xDiff)(const char*,FILE*) = diff_one_table; +#ifndef SQLITE_OMIT_LOAD_EXTENSION + int nExt = 0; + char **azExt = 0; +#endif + int useTransaction = 0; + int neverUseTransaction = 0; + + g.zArgv0 = argv[0]; + sqlite3_config(SQLITE_CONFIG_SINGLETHREAD); + for(i=1; i<argc; i++){ + const char *z = argv[i]; + if( z[0]=='-' ){ + z++; + if( z[0]=='-' ) z++; + if( strcmp(z,"changeset")==0 ){ + if( i==argc-1 ) cmdlineError("missing argument to %s", argv[i]); + out = fopen(argv[++i], "wb"); + if( out==0 ) cmdlineError("cannot open: %s", argv[i]); + xDiff = changeset_one_table; + neverUseTransaction = 1; + }else + if( strcmp(z,"debug")==0 ){ + if( i==argc-1 ) cmdlineError("missing argument to %s", argv[i]); + g.fDebug = strtol(argv[++i], 0, 0); + }else + if( strcmp(z,"help")==0 ){ + showHelp(); + return 0; + }else +#ifndef SQLITE_OMIT_LOAD_EXTENSION + if( strcmp(z,"lib")==0 || strcmp(z,"L")==0 ){ + if( i==argc-1 ) cmdlineError("missing argument to %s", argv[i]); + azExt = realloc(azExt, sizeof(azExt[0])*(nExt+1)); + if( azExt==0 ) cmdlineError("out of memory"); + azExt[nExt++] = argv[++i]; + }else +#endif + if( strcmp(z,"primarykey")==0 ){ + g.bSchemaPK = 1; + }else + if( strcmp(z,"rbu")==0 ){ + xDiff = rbudiff_one_table; + }else + if( strcmp(z,"schema")==0 ){ + g.bSchemaOnly = 1; + }else + if( strcmp(z,"summary")==0 ){ + xDiff = summarize_one_table; + }else + if( strcmp(z,"table")==0 ){ + if( i==argc-1 ) cmdlineError("missing argument to %s", argv[i]); + zTab = argv[++i]; + g.bSchemaCompare = + sqlite3_stricmp(zTab, "sqlite_schema")==0 + || sqlite3_stricmp(zTab, "sqlite_master")==0; + }else + if( strcmp(z,"transaction")==0 ){ + useTransaction = 1; + }else + if( strcmp(z,"vtab")==0 ){ + g.bHandleVtab = 1; + }else + { + cmdlineError("unknown option: %s", argv[i]); + } + }else if( zDb1==0 ){ + zDb1 = argv[i]; + }else if( zDb2==0 ){ + zDb2 = argv[i]; + }else{ + cmdlineError("unknown argument: %s", argv[i]); + } + } + if( zDb2==0 ){ + cmdlineError("two database arguments required"); + } + if( g.bSchemaOnly && g.bSchemaCompare ){ + cmdlineError("The --schema option is useless with --table %s .", zTab); + } + rc = sqlite3_open(zDb1, &g.db); + if( rc ){ + cmdlineError("cannot open database file \"%s\"", zDb1); + } + rc = sqlite3_exec(g.db, "SELECT * FROM sqlite_schema", 0, 0, &zErrMsg); + if( rc || zErrMsg ){ + cmdlineError("\"%s\" does not appear to be a valid SQLite database", zDb1); + } +#ifndef SQLITE_OMIT_LOAD_EXTENSION + sqlite3_enable_load_extension(g.db, 1); + for(i=0; i<nExt; i++){ + rc = sqlite3_load_extension(g.db, azExt[i], 0, &zErrMsg); + if( rc || zErrMsg ){ + cmdlineError("error loading %s: %s", azExt[i], zErrMsg); + } + } + free(azExt); +#endif + zSql = sqlite3_mprintf("ATTACH %Q as aux;", zDb2); + rc = sqlite3_exec(g.db, zSql, 0, 0, &zErrMsg); + sqlite3_free(zSql); + zSql = 0; + if( rc || zErrMsg ){ + cmdlineError("cannot attach database \"%s\"", zDb2); + } + rc = sqlite3_exec(g.db, "SELECT * FROM aux.sqlite_schema", 0, 0, &zErrMsg); + if( rc || zErrMsg ){ + cmdlineError("\"%s\" does not appear to be a valid SQLite database", zDb2); + } + + if( neverUseTransaction ) useTransaction = 0; + if( useTransaction ) Wfprintf(out, "BEGIN TRANSACTION;\n"); + if( xDiff==rbudiff_one_table ){ + Wfprintf(out, "CREATE TABLE IF NOT EXISTS rbu_count" + "(tbl TEXT PRIMARY KEY COLLATE NOCASE, cnt INTEGER) " + "WITHOUT ROWID;\n" + ); + } + if( zTab ){ + xDiff(zTab, out); + }else{ + /* Handle tables one by one */ + pStmt = db_prepare("%s", all_tables_sql() ); + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + xDiff((const char*)sqlite3_column_text(pStmt,0), out); + } + sqlite3_finalize(pStmt); + } + if( useTransaction ) Wfprintf(stdout,"COMMIT;\n"); + + /* TBD: Handle trigger differences */ + /* TBD: Handle view differences */ + sqlite3_close(g.db); + return 0; +} diff --git a/tool/sqlite3_analyzer.c.in b/tool/sqlite3_analyzer.c.in new file mode 100644 index 0000000..2d799ed --- /dev/null +++ b/tool/sqlite3_analyzer.c.in @@ -0,0 +1,76 @@ +/* +** Read an SQLite database file and analyze its space utilization. Generate +** text on standard output. +*/ +#define TCLSH_INIT_PROC sqlite3_analyzer_init_proc +#define SQLITE_ENABLE_DBSTAT_VTAB 1 +#undef SQLITE_THREADSAFE +#define SQLITE_THREADSAFE 0 +#undef SQLITE_ENABLE_COLUMN_METADATA +#define SQLITE_OMIT_DECLTYPE 1 +#define SQLITE_OMIT_DEPRECATED 1 +#define SQLITE_OMIT_PROGRESS_CALLBACK 1 +#define SQLITE_OMIT_SHARED_CACHE 1 +#define SQLITE_DEFAULT_MEMSTATUS 0 +#define SQLITE_MAX_EXPR_DEPTH 0 +#define SQLITE_OMIT_LOAD_EXTENSION 1 +#if !defined(SQLITE_AMALGAMATION) && !defined(USE_EXTERNAL_SQLITE) +INCLUDE sqlite3.c +#endif +INCLUDE $ROOT/src/tclsqlite.c + +#if defined(_WIN32) +INCLUDE $ROOT/ext/consio/console_io.h +INCLUDE $ROOT/ext/consio/console_io.c + +/* Substitute "puts" command. Only these forms recognized: +** +** puts STRING +** puts stderr STRING +** puts -nonewline STRING +*/ +static int subst_puts( + void *NotUsed, + Tcl_Interp *interp, + int objc, + Tcl_Obj *const*objv +){ + FILE *pOut = stdout; + const char *zOut; + int addNewLine = 1; + if( objc==2 ){ + zOut = Tcl_GetString(objv[1]); + }else if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "?stderr|-nonewline? STRING"); + return TCL_ERROR; + }else{ + const char *zArg = Tcl_GetString(objv[1]); + if( zArg==0 ) return TCL_ERROR; + zOut = Tcl_GetString(objv[2]); + if( strcmp(zArg, "stderr")==0 ){ + pOut = stderr; + }else if( strcmp(zArg, "-nonewline")==0 ){ + addNewLine = 0; + }else{ + Tcl_AppendResult(interp, "bad argument: ", zArg, 0); + return TCL_ERROR; + } + } + fPutsUtf8(zOut, pOut); + if( addNewLine ) fPutsUtf8("\n", pOut); + return TCL_OK; +} +#endif /* defined(_WIN32) */ + +const char *sqlite3_analyzer_init_proc(Tcl_Interp *interp){ +#if defined(_WIN32) + Tcl_CreateObjCommand(interp, "puts", subst_puts, 0, 0); +#else + (void)interp; +#endif + return +BEGIN_STRING +INCLUDE $ROOT/tool/spaceanal.tcl +END_STRING +; +} diff --git a/tool/sqltclsh.c.in b/tool/sqltclsh.c.in new file mode 100644 index 0000000..da354ee --- /dev/null +++ b/tool/sqltclsh.c.in @@ -0,0 +1,51 @@ +/* +** This is the source code to a "tclsh" that has SQLite built-in. +** +** The startup script is located as follows: +** +** (1) Open the executable as an appended SQLite database and try to +** read the startup script out of that database. +** +** (2) If the first argument is a readable file, try to open that file +** as an SQLite database and read the startup script out of that +** database. +** +** (3) If the first argument is a readable file with a ".tcl" extension, +** then try to run that script directly. +** +** If none of the above steps work, then the program runs as an interactive +** tclsh. +*/ +#define TCLSH_INIT_PROC sqlite3_tclapp_init_proc +#define SQLITE_ENABLE_DBSTAT_VTAB 1 +#undef SQLITE_THREADSAFE +#define SQLITE_THREADSAFE 0 +#undef SQLITE_ENABLE_COLUMN_METADATA +#define SQLITE_OMIT_DECLTYPE 1 +#define SQLITE_OMIT_DEPRECATED 1 +#define SQLITE_OMIT_PROGRESS_CALLBACK 1 +#define SQLITE_OMIT_SHARED_CACHE 1 +#define SQLITE_DEFAULT_MEMSTATUS 0 +#define SQLITE_MAX_EXPR_DEPTH 0 +INCLUDE sqlite3.c +INCLUDE $ROOT/ext/misc/appendvfs.c +#ifdef SQLITE_HAVE_ZLIB +INCLUDE $ROOT/ext/misc/zipfile.c +INCLUDE $ROOT/ext/misc/sqlar.c +#endif +INCLUDE $ROOT/src/tclsqlite.c + +const char *sqlite3_tclapp_init_proc(Tcl_Interp *interp){ + (void)interp; + sqlite3_appendvfs_init(0,0,0); +#ifdef SQLITE_HAVE_ZLIB + sqlite3_auto_extension((void(*)(void))sqlite3_sqlar_init); + sqlite3_auto_extension((void(*)(void))sqlite3_zipfile_init); +#endif + + return +BEGIN_STRING +INCLUDE $ROOT/tool/sqltclsh.tcl +END_STRING +; +} diff --git a/tool/sqltclsh.tcl b/tool/sqltclsh.tcl new file mode 100644 index 0000000..6a4b1fe --- /dev/null +++ b/tool/sqltclsh.tcl @@ -0,0 +1,71 @@ +# Try to open the executable as a database and read the "scripts.data" +# field where "scripts.name" is 'main.tcl' +# +catch { + if {![file exists $argv0] && [file exists $argv0.exe]} { + append argv0 .exe + } + sqlite3 db $argv0 -vfs apndvfs -create 0 + set mainscript [db one { + SELECT sqlar_uncompress(data,sz) FROM sqlar WHERE name='main.tcl' + }] +} +if {[info exists mainscript]} { + eval $mainscript + return +} else { + catch {db close} +} + +# Try to open file named in the first argument as a database and +# read the "scripts.data" field where "scripts.name" is 'main.tcl' +# +if {[llength $argv]>0 && [file readable [lindex $argv 0]]} { + catch { + sqlite3 db [lindex $argv 0] -vfs apndvfs -create 0 + set mainscript [db one {SELECT data FROM scripts WHERE name='main.tcl'}] + set argv0 [lindex $argv 0] + set argv [lrange $argv 1 end] + } + if {[info exists mainscript]} { + eval $mainscript + return + } else { + catch {db close} + } + if {[string match *.tcl [lindex $argv 0]]} { + set fd [open [lindex $argv 0] rb] + set mainscript [read $fd] + close $fd + unset fd + set argv0 [lindex $argv 0] + set argv [lrange $argv 1 end] + } + if {[info exists mainscript]} { + eval $mainscript + return + } +} + +# If all else fails, do an interactive loop +# +set line {} +while {![eof stdin]} { + if {$line!=""} { + puts -nonewline "> " + } else { + puts -nonewline "% " + } + flush stdout + append line [gets stdin] + if {[info complete $line]} { + if {[catch {uplevel #0 $line} result]} { + puts stderr "Error: $result" + } elseif {$result!=""} { + puts $result + } + set line {} + } else { + append line \\n" + } +} diff --git a/tool/src-verify.c b/tool/src-verify.c new file mode 100644 index 0000000..7629046 --- /dev/null +++ b/tool/src-verify.c @@ -0,0 +1,956 @@ +/* +** This utility program reads the "manifest" and "manifest.uuid" files +** in a Fossil-generated source tree (where the repository has the +** "manifest" setting turned on - this is true for SQLite and Fossil itself) +** and verifies that the source code files are complete and unaltered by +** checking the SHA1 and SHA3 hashes of the source files contained in the +** "manifest" file. +** +** On success it prints: "OK $HASH" where $HASH is the SHA3-256 hash of +** the check-in for the source tree. If it finds any discrepencies, it +** prints "Derived from $HASH with changes to:" followed by a list of files +** which have been altered. +** +** USAGE: +** +** src-verify [-x] [-v] $(ROOT) +** +** Where ROOT is the root of the source tree - the directory that contains +** the "manifest" and "manifest.uuid" files. Add the "-v" option for +** some debugging output. With the -x option, the output is in a format +** that is intended to be read by a script rather by a human. The -x output +** format always has the SHA3 hash of the source check-in on the first line +** and lists files that have changed on subsequent lines. +** +** Additional debugging options: +** +** src-verify --sha1 FILE ... +** src-verify --sha3 FILE ... +** +** Compute the SHA1 or SHA3-256 hashes for all of the FILEs named +** +** COMPILING: +** +** This utility is self-contained. It uses only the standard library. +** There are no other dependencies. Just compile it and run it. +** +** LIMITATIONS: +** +** * This utility assumes that the check-in hash uses SHA3-256. +** It is ok for individual file hashes to be SHA1, but the +** check-in itself must use a SHA3-256 hash. +*/ +#include <stdio.h> +#include <string.h> +#include <stdlib.h> +#if !defined(_WIN32) +# include <unistd.h> +#else +# include <io.h> +# ifndef R_OK +# define R_OK 04 +# endif +# ifndef access +# define access(f,m) _access((f),(m)) +# endif +#endif +typedef unsigned long long int u64; + +/* +** The SHA1 implementation below is adapted from: +** +** $NetBSD: sha1.c,v 1.6 2009/11/06 20:31:18 joerg Exp $ +** $OpenBSD: sha1.c,v 1.9 1997/07/23 21:12:32 kstailey Exp $ +** +** SHA-1 in C +** By Steve Reid <steve@edmweb.com> +** 100% Public Domain +*/ +typedef struct SHA1Context SHA1Context; +struct SHA1Context { + unsigned int state[5]; + unsigned int count[2]; + unsigned char buffer[64]; +}; + +/* + * blk0() and blk() perform the initial expand. + * I got the idea of expanding during the round function from SSLeay + * + * blk0le() for little-endian and blk0be() for big-endian. + */ +#define SHA_ROT(x,l,r) ((x) << (l) | (x) >> (r)) +#define rol(x,k) SHA_ROT(x,k,32-(k)) +#define ror(x,k) SHA_ROT(x,32-(k),k) +#define blk0le(i) (block[i] = (ror(block[i],8)&0xFF00FF00) \ + |(rol(block[i],8)&0x00FF00FF)) +#define blk0be(i) block[i] +#define blk(i) (block[i&15] = rol(block[(i+13)&15]^block[(i+8)&15] \ + ^block[(i+2)&15]^block[i&15],1)) + +/* + * (R0+R1), R2, R3, R4 are the different operations (rounds) used in SHA1 + * + * Rl0() for little-endian and Rb0() for big-endian. Endianness is + * determined at run-time. + */ +#define Rl0(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk0le(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define Rb0(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk0be(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define R1(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define R2(v,w,x,y,z,i) \ + z+=(w^x^y)+blk(i)+0x6ED9EBA1+rol(v,5);w=ror(w,2); +#define R3(v,w,x,y,z,i) \ + z+=(((w|x)&y)|(w&x))+blk(i)+0x8F1BBCDC+rol(v,5);w=ror(w,2); +#define R4(v,w,x,y,z,i) \ + z+=(w^x^y)+blk(i)+0xCA62C1D6+rol(v,5);w=ror(w,2); + +/* + * Hash a single 512-bit block. This is the core of the algorithm. + */ +#define a qq[0] +#define b qq[1] +#define c qq[2] +#define d qq[3] +#define e qq[4] + +void SHA1Transform(unsigned int state[5], const unsigned char buffer[64]) +{ + unsigned int qq[5]; /* a, b, c, d, e; */ + static int one = 1; + unsigned int block[16]; + memcpy(block, buffer, 64); + memcpy(qq,state,5*sizeof(unsigned int)); + + /* Copy context->state[] to working vars */ + /* + a = state[0]; + b = state[1]; + c = state[2]; + d = state[3]; + e = state[4]; + */ + + /* 4 rounds of 20 operations each. Loop unrolled. */ + if( 1 == *(unsigned char*)&one ){ + Rl0(a,b,c,d,e, 0); Rl0(e,a,b,c,d, 1); Rl0(d,e,a,b,c, 2); Rl0(c,d,e,a,b, 3); + Rl0(b,c,d,e,a, 4); Rl0(a,b,c,d,e, 5); Rl0(e,a,b,c,d, 6); Rl0(d,e,a,b,c, 7); + Rl0(c,d,e,a,b, 8); Rl0(b,c,d,e,a, 9); Rl0(a,b,c,d,e,10); Rl0(e,a,b,c,d,11); + Rl0(d,e,a,b,c,12); Rl0(c,d,e,a,b,13); Rl0(b,c,d,e,a,14); Rl0(a,b,c,d,e,15); + }else{ + Rb0(a,b,c,d,e, 0); Rb0(e,a,b,c,d, 1); Rb0(d,e,a,b,c, 2); Rb0(c,d,e,a,b, 3); + Rb0(b,c,d,e,a, 4); Rb0(a,b,c,d,e, 5); Rb0(e,a,b,c,d, 6); Rb0(d,e,a,b,c, 7); + Rb0(c,d,e,a,b, 8); Rb0(b,c,d,e,a, 9); Rb0(a,b,c,d,e,10); Rb0(e,a,b,c,d,11); + Rb0(d,e,a,b,c,12); Rb0(c,d,e,a,b,13); Rb0(b,c,d,e,a,14); Rb0(a,b,c,d,e,15); + } + R1(e,a,b,c,d,16); R1(d,e,a,b,c,17); R1(c,d,e,a,b,18); R1(b,c,d,e,a,19); + R2(a,b,c,d,e,20); R2(e,a,b,c,d,21); R2(d,e,a,b,c,22); R2(c,d,e,a,b,23); + R2(b,c,d,e,a,24); R2(a,b,c,d,e,25); R2(e,a,b,c,d,26); R2(d,e,a,b,c,27); + R2(c,d,e,a,b,28); R2(b,c,d,e,a,29); R2(a,b,c,d,e,30); R2(e,a,b,c,d,31); + R2(d,e,a,b,c,32); R2(c,d,e,a,b,33); R2(b,c,d,e,a,34); R2(a,b,c,d,e,35); + R2(e,a,b,c,d,36); R2(d,e,a,b,c,37); R2(c,d,e,a,b,38); R2(b,c,d,e,a,39); + R3(a,b,c,d,e,40); R3(e,a,b,c,d,41); R3(d,e,a,b,c,42); R3(c,d,e,a,b,43); + R3(b,c,d,e,a,44); R3(a,b,c,d,e,45); R3(e,a,b,c,d,46); R3(d,e,a,b,c,47); + R3(c,d,e,a,b,48); R3(b,c,d,e,a,49); R3(a,b,c,d,e,50); R3(e,a,b,c,d,51); + R3(d,e,a,b,c,52); R3(c,d,e,a,b,53); R3(b,c,d,e,a,54); R3(a,b,c,d,e,55); + R3(e,a,b,c,d,56); R3(d,e,a,b,c,57); R3(c,d,e,a,b,58); R3(b,c,d,e,a,59); + R4(a,b,c,d,e,60); R4(e,a,b,c,d,61); R4(d,e,a,b,c,62); R4(c,d,e,a,b,63); + R4(b,c,d,e,a,64); R4(a,b,c,d,e,65); R4(e,a,b,c,d,66); R4(d,e,a,b,c,67); + R4(c,d,e,a,b,68); R4(b,c,d,e,a,69); R4(a,b,c,d,e,70); R4(e,a,b,c,d,71); + R4(d,e,a,b,c,72); R4(c,d,e,a,b,73); R4(b,c,d,e,a,74); R4(a,b,c,d,e,75); + R4(e,a,b,c,d,76); R4(d,e,a,b,c,77); R4(c,d,e,a,b,78); R4(b,c,d,e,a,79); + + /* Add the working vars back into context.state[] */ + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; + state[4] += e; +} + +/* + * SHA1Init - Initialize new context + */ +static void SHA1Init(SHA1Context *context){ + /* SHA1 initialization constants */ + context->state[0] = 0x67452301; + context->state[1] = 0xEFCDAB89; + context->state[2] = 0x98BADCFE; + context->state[3] = 0x10325476; + context->state[4] = 0xC3D2E1F0; + context->count[0] = context->count[1] = 0; +} + + +/* + * Run your data through this. + */ +static void SHA1Update( + SHA1Context *context, + const unsigned char *data, + unsigned int len +){ + unsigned int i, j; + + j = context->count[0]; + if ((context->count[0] += len << 3) < j) + context->count[1] += (len>>29)+1; + j = (j >> 3) & 63; + if ((j + len) > 63) { + (void)memcpy(&context->buffer[j], data, (i = 64-j)); + SHA1Transform(context->state, context->buffer); + for ( ; i + 63 < len; i += 64) + SHA1Transform(context->state, &data[i]); + j = 0; + } else { + i = 0; + } + (void)memcpy(&context->buffer[j], &data[i], len - i); +} + + +/* + * Add padding and return the message digest. + */ +static void SHA1Final(unsigned char *digest, SHA1Context *context){ + unsigned int i; + unsigned char finalcount[8]; + + for (i = 0; i < 8; i++) { + finalcount[i] = (unsigned char)((context->count[(i >= 4 ? 0 : 1)] + >> ((3-(i & 3)) * 8) ) & 255); /* Endian independent */ + } + SHA1Update(context, (const unsigned char *)"\200", 1); + while ((context->count[0] & 504) != 448) + SHA1Update(context, (const unsigned char *)"\0", 1); + SHA1Update(context, finalcount, 8); /* Should cause a SHA1Transform() */ + + if (digest) { + for (i = 0; i < 20; i++) + digest[i] = (unsigned char) + ((context->state[i>>2] >> ((3-(i & 3)) * 8) ) & 255); + } +} + + +/* +** Macros to determine whether the machine is big or little endian, +** and whether or not that determination is run-time or compile-time. +** +** For best performance, an attempt is made to guess at the byte-order +** using C-preprocessor macros. If that is unsuccessful, or if +** -DSHA3_BYTEORDER=0 is set, then byte-order is determined +** at run-time. +*/ +#ifndef SHA3_BYTEORDER +# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__arm__) +# define SHA3_BYTEORDER 1234 +# elif defined(sparc) || defined(__ppc__) +# define SHA3_BYTEORDER 4321 +# else +# define SHA3_BYTEORDER 0 +# endif +#endif + + +/* +** State structure for a SHA3 hash in progress +*/ +typedef struct SHA3Context SHA3Context; +struct SHA3Context { + union { + u64 s[25]; /* Keccak state. 5x5 lines of 64 bits each */ + unsigned char x[1600]; /* ... or 1600 bytes */ + } u; + unsigned nRate; /* Bytes of input accepted per Keccak iteration */ + unsigned nLoaded; /* Input bytes loaded into u.x[] so far this cycle */ + unsigned ixMask; /* Insert next input into u.x[nLoaded^ixMask]. */ +}; + +/* +** A single step of the Keccak mixing function for a 1600-bit state +*/ +static void KeccakF1600Step(SHA3Context *p){ + int i; + u64 B0, B1, B2, B3, B4; + u64 C0, C1, C2, C3, C4; + u64 D0, D1, D2, D3, D4; + static const u64 RC[] = { + 0x0000000000000001ULL, 0x0000000000008082ULL, + 0x800000000000808aULL, 0x8000000080008000ULL, + 0x000000000000808bULL, 0x0000000080000001ULL, + 0x8000000080008081ULL, 0x8000000000008009ULL, + 0x000000000000008aULL, 0x0000000000000088ULL, + 0x0000000080008009ULL, 0x000000008000000aULL, + 0x000000008000808bULL, 0x800000000000008bULL, + 0x8000000000008089ULL, 0x8000000000008003ULL, + 0x8000000000008002ULL, 0x8000000000000080ULL, + 0x000000000000800aULL, 0x800000008000000aULL, + 0x8000000080008081ULL, 0x8000000000008080ULL, + 0x0000000080000001ULL, 0x8000000080008008ULL + }; +# define A00 (p->u.s[0]) +# define A01 (p->u.s[1]) +# define A02 (p->u.s[2]) +# define A03 (p->u.s[3]) +# define A04 (p->u.s[4]) +# define A10 (p->u.s[5]) +# define A11 (p->u.s[6]) +# define A12 (p->u.s[7]) +# define A13 (p->u.s[8]) +# define A14 (p->u.s[9]) +# define A20 (p->u.s[10]) +# define A21 (p->u.s[11]) +# define A22 (p->u.s[12]) +# define A23 (p->u.s[13]) +# define A24 (p->u.s[14]) +# define A30 (p->u.s[15]) +# define A31 (p->u.s[16]) +# define A32 (p->u.s[17]) +# define A33 (p->u.s[18]) +# define A34 (p->u.s[19]) +# define A40 (p->u.s[20]) +# define A41 (p->u.s[21]) +# define A42 (p->u.s[22]) +# define A43 (p->u.s[23]) +# define A44 (p->u.s[24]) +# define ROL64(a,x) ((a<<x)|(a>>(64-x))) + + for(i=0; i<24; i+=4){ + C0 = A00^A10^A20^A30^A40; + C1 = A01^A11^A21^A31^A41; + C2 = A02^A12^A22^A32^A42; + C3 = A03^A13^A23^A33^A43; + C4 = A04^A14^A24^A34^A44; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A11^D1), 44); + B2 = ROL64((A22^D2), 43); + B3 = ROL64((A33^D3), 21); + B4 = ROL64((A44^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i]; + A11 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A20^D0), 3); + B3 = ROL64((A31^D1), 45); + B4 = ROL64((A42^D2), 61); + B0 = ROL64((A03^D3), 28); + B1 = ROL64((A14^D4), 20); + A20 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A40^D0), 18); + B0 = ROL64((A01^D1), 1); + B1 = ROL64((A12^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A34^D4), 8); + A40 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A10^D0), 36); + B2 = ROL64((A21^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A43^D3), 56); + B0 = ROL64((A04^D4), 27); + A10 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A30^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A02^D2), 62); + B1 = ROL64((A13^D3), 55); + B2 = ROL64((A24^D4), 39); + A30 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + C0 = A00^A20^A40^A10^A30; + C1 = A11^A31^A01^A21^A41; + C2 = A22^A42^A12^A32^A02; + C3 = A33^A03^A23^A43^A13; + C4 = A44^A14^A34^A04^A24; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A31^D1), 44); + B2 = ROL64((A12^D2), 43); + B3 = ROL64((A43^D3), 21); + B4 = ROL64((A24^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+1]; + A31 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A40^D0), 3); + B3 = ROL64((A21^D1), 45); + B4 = ROL64((A02^D2), 61); + B0 = ROL64((A33^D3), 28); + B1 = ROL64((A14^D4), 20); + A40 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A30^D0), 18); + B0 = ROL64((A11^D1), 1); + B1 = ROL64((A42^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A04^D4), 8); + A30 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A20^D0), 36); + B2 = ROL64((A01^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A13^D3), 56); + B0 = ROL64((A44^D4), 27); + A20 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A10^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A22^D2), 62); + B1 = ROL64((A03^D3), 55); + B2 = ROL64((A34^D4), 39); + A10 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + C0 = A00^A40^A30^A20^A10; + C1 = A31^A21^A11^A01^A41; + C2 = A12^A02^A42^A32^A22; + C3 = A43^A33^A23^A13^A03; + C4 = A24^A14^A04^A44^A34; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A21^D1), 44); + B2 = ROL64((A42^D2), 43); + B3 = ROL64((A13^D3), 21); + B4 = ROL64((A34^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+2]; + A21 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A30^D0), 3); + B3 = ROL64((A01^D1), 45); + B4 = ROL64((A22^D2), 61); + B0 = ROL64((A43^D3), 28); + B1 = ROL64((A14^D4), 20); + A30 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A10^D0), 18); + B0 = ROL64((A31^D1), 1); + B1 = ROL64((A02^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A44^D4), 8); + A10 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A40^D0), 36); + B2 = ROL64((A11^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A03^D3), 56); + B0 = ROL64((A24^D4), 27); + A40 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A20^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A12^D2), 62); + B1 = ROL64((A33^D3), 55); + B2 = ROL64((A04^D4), 39); + A20 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + C0 = A00^A30^A10^A40^A20; + C1 = A21^A01^A31^A11^A41; + C2 = A42^A22^A02^A32^A12; + C3 = A13^A43^A23^A03^A33; + C4 = A34^A14^A44^A24^A04; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A01^D1), 44); + B2 = ROL64((A02^D2), 43); + B3 = ROL64((A03^D3), 21); + B4 = ROL64((A04^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+3]; + A01 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A10^D0), 3); + B3 = ROL64((A11^D1), 45); + B4 = ROL64((A12^D2), 61); + B0 = ROL64((A13^D3), 28); + B1 = ROL64((A14^D4), 20); + A10 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A20^D0), 18); + B0 = ROL64((A21^D1), 1); + B1 = ROL64((A22^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A24^D4), 8); + A20 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A30^D0), 36); + B2 = ROL64((A31^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A33^D3), 56); + B0 = ROL64((A34^D4), 27); + A30 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A40^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A42^D2), 62); + B1 = ROL64((A43^D3), 55); + B2 = ROL64((A44^D4), 39); + A40 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + } +} + +/* +** Initialize a new hash. iSize determines the size of the hash +** in bits and should be one of 224, 256, 384, or 512. Or iSize +** can be zero to use the default hash size of 256 bits. +*/ +static void SHA3Init(SHA3Context *p, int iSize){ + memset(p, 0, sizeof(*p)); + if( iSize>=128 && iSize<=512 ){ + p->nRate = (1600 - ((iSize + 31)&~31)*2)/8; + }else{ + p->nRate = (1600 - 2*256)/8; + } +#if SHA3_BYTEORDER==1234 + /* Known to be little-endian at compile-time. No-op */ +#elif SHA3_BYTEORDER==4321 + p->ixMask = 7; /* Big-endian */ +#else + { + static unsigned int one = 1; + if( 1==*(unsigned char*)&one ){ + /* Little endian. No byte swapping. */ + p->ixMask = 0; + }else{ + /* Big endian. Byte swap. */ + p->ixMask = 7; + } + } +#endif +} + +/* +** Make consecutive calls to the SHA3Update function to add new content +** to the hash +*/ +static void SHA3Update( + SHA3Context *p, + const unsigned char *aData, + unsigned int nData +){ + unsigned int i = 0; +#if SHA3_BYTEORDER==1234 + if( (p->nLoaded % 8)==0 && ((aData - (const unsigned char*)0)&7)==0 ){ + for(; i+7<nData; i+=8){ + p->u.s[p->nLoaded/8] ^= *(u64*)&aData[i]; + p->nLoaded += 8; + if( p->nLoaded>=p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } + } +#endif + for(; i<nData; i++){ +#if SHA3_BYTEORDER==1234 + p->u.x[p->nLoaded] ^= aData[i]; +#elif SHA3_BYTEORDER==4321 + p->u.x[p->nLoaded^0x07] ^= aData[i]; +#else + p->u.x[p->nLoaded^p->ixMask] ^= aData[i]; +#endif + p->nLoaded++; + if( p->nLoaded==p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } +} + +/* +** After all content has been added, invoke SHA3Final() to compute +** the final hash. The function returns a pointer to the binary +** hash value. +*/ +static unsigned char *SHA3Final(SHA3Context *p){ + unsigned int i; + if( p->nLoaded==p->nRate-1 ){ + const unsigned char c1 = 0x86; + SHA3Update(p, &c1, 1); + }else{ + const unsigned char c2 = 0x06; + const unsigned char c3 = 0x80; + SHA3Update(p, &c2, 1); + p->nLoaded = p->nRate - 1; + SHA3Update(p, &c3, 1); + } + for(i=0; i<p->nRate; i++){ + p->u.x[i+p->nRate] = p->u.x[i^p->ixMask]; + } + return &p->u.x[p->nRate]; +} + +/* +** Convert a digest into base-16. +*/ +static void DigestToBase16(unsigned char *digest, char *zBuf, int nByte){ + static const char zEncode[] = "0123456789abcdef"; + int ix; + + for(ix=0; ix<nByte; ix++){ + *zBuf++ = zEncode[(*digest>>4)&0xf]; + *zBuf++ = zEncode[*digest++ & 0xf]; + } + *zBuf = '\0'; +} + +/* +** Compute the SHA3-256 checksum of a file on disk. Store the resulting +** checksum in the zCksum. +** +** Return the number of errors. +*/ +void sha3sum_file(const char *zFilename, char *zCksum){ + FILE *in; + SHA3Context ctx; + char zBuf[10240]; + + in = fopen(zFilename,"rb"); + if( in==0 ){ + zCksum[0] = 0; + return; + } + SHA3Init(&ctx, 256); + for(;;){ + size_t n; + n = fread(zBuf, 1, sizeof(zBuf), in); + if( n<=0 ) break; + SHA3Update(&ctx, (unsigned char*)zBuf, (unsigned)n); + } + fclose(in); + DigestToBase16(SHA3Final(&ctx), zCksum, 32); +} + +/* +** Compute the SHA1 checksum of a file on disk. Store the resulting +** checksum in the zCksum. +** +** Return the number of errors. +*/ +void sha1sum_file(const char *zFilename, char *zCksum){ + FILE *in; + SHA1Context ctx; + unsigned char zResult[20]; + char zBuf[10240]; + + in = fopen(zFilename,"rb"); + if( in==0 ){ + zCksum[0] = 0; + return; + } + SHA1Init(&ctx); + for(;;){ + size_t n; + n = fread(zBuf, 1, sizeof(zBuf), in); + if( n<=0 ) break; + SHA1Update(&ctx, (unsigned char*)zBuf, (unsigned)n); + } + fclose(in); + SHA1Final(zResult, &ctx); + DigestToBase16(zResult, zCksum, 20); +} + +/* +** Decode a fossilized string in-place. +*/ +void defossilize(char *z){ + int i, j, cc; + char *zSlash = strchr(z, '\\'); + if( zSlash==0 ) return; + i = zSlash - z; + for(j=i; (cc=z[i])!=0; i++){ + if( cc=='\\' && z[i+1] ){ + i++; + switch( z[i] ){ + case 'n': cc = '\n'; break; + case 's': cc = ' '; break; + case 't': cc = '\t'; break; + case 'r': cc = '\r'; break; + case 'v': cc = '\v'; break; + case 'f': cc = '\f'; break; + case '0': cc = 0; break; + case '\\': cc = '\\'; break; + default: cc = z[i]; break; + } + } + z[j++] = cc; + } + if( z[j] ) z[j] = 0; +} + +/* +** Report that a single file is incorrect. +*/ +static void errorMsg(int *pnErr, const char *zVers, const char *zFile){ + if( *pnErr==0 ){ + printf("Derived from %.25s with changes to:\n", zVers); + } + printf(" %s\n", zFile); + (*pnErr)++; +} +static void errorMsgNH(int *pnErr, const char *zVers, const char *zFile){ + if( *pnErr==0 ){ + printf("%s\n", zVers); + } + printf("%s\n", zFile); + (*pnErr)++; +} + +int main(int argc, char **argv){ + int i, j; + int nDir; + FILE *in; + int bDebug = 0; + int bNonHuman = 0; + int bSeenManifestErr = 0; + int nErr = 0; + SHA3Context ctx3; + const char *zDir = 0; + void (*xErr)(int*,const char*,const char*); + char zHash[100]; + char zCk[100]; + char zVers[100]; + char zLine[40000]; + char zFile[40000]; + xErr = errorMsg; + for(i=1; i<argc; i++){ + const char *z = argv[i]; + if( z[0]!='-' ){ + if( zDir!=0 ){ + fprintf(stderr, "bad argument: %s\n", z); + return 1; + } + zDir = z; + continue; + } + if( z[1]=='-' && z[2]!=0 ) z++; + if( strcmp(argv[1],"-sha1")==0 ){ + /* For testing purposes, if the first argument is --sha1, then simply + ** compute and print the SHA1 checksum of all subsequent arguments. */ + for(i++; i<argc; i++){ + sha1sum_file(argv[i], zHash); + printf("%s %s\n", zHash, argv[i]); + } + return 0; + } + if( strcmp(argv[1], "-sha3")==0 ){ + /* For testing purposes, if the first argument is --sha3, then simply + ** compute and print the SHA3-256 checksum of all subsequent arguments. */ + for(i++; i<argc; i++){ + sha3sum_file(argv[i], zHash); + printf("%s %s\n", zHash, argv[i]); + } + return 0; + } + if( strcmp(z,"-v")==0 ){ + bDebug = 1; + continue; + } + if( strcmp(z,"-x")==0 ){ + bNonHuman = 1; + xErr = errorMsgNH; + continue; + } + fprintf(stderr, "Usage: %s DIRECTORY\n" + " or: %s --sha1 FILE ...\n" + " or: %s --sha3 FILE ...\n", + argv[0], argv[0], argv[0]); + return 1; + } + if( strlen(zDir)>1000 ){ + fprintf(stderr, "Directory argument too big: [%s]\n", zDir); + return 1; + } + nDir = (int)strlen(zDir); + if( nDir<0 ){ + fprintf(stderr, "Directory argument too short.\n"); + return 1; + } + memcpy(zFile, zDir, nDir); + if( zFile[nDir-1]!='/' ){ + zFile[nDir++] = '/'; + } + memcpy(&zFile[nDir], "manifest", 9); + if( bDebug ){ + printf("manifest file: [%s]\n", zFile); + } + in = fopen(zFile, "rb"); + if( in==0 ){ + fprintf(stderr, "missing manifest: \"%s\"\n", zFile); + return 1; + } + SHA3Init(&ctx3, 256); + while( fgets(zLine, sizeof(zLine), in) ){ + if( zLine[0]=='#' ) break; + SHA3Update(&ctx3, (unsigned char*)zLine, (int)strlen(zLine)); + } + DigestToBase16(SHA3Final(&ctx3), zVers, 32); + + rewind(in); + while( fgets(zLine, sizeof(zLine), in) ){ + if( zLine[0]!='F' ) continue; + if( zLine[1]!=' ' ) continue; + for(i=2, j=nDir; zLine[i]!=0 && zLine[i]!=' '; i++, j++){ + if( j<sizeof(zFile) ) zFile[j] = zLine[i]; + } + if( j<sizeof(zFile) ) zFile[j] = 0; + zFile[sizeof(zFile)-1] = 0; + defossilize(&zFile[nDir]); + if( zLine[i]!=' ' ){ + bSeenManifestErr = 1; + continue; + } + for(i++, j=0; zLine[i]>='0' && zLine[i]<='f'; i++, j++){ + if( j<sizeof(zHash) ) zHash[j] = zLine[i]; + } + if( j<sizeof(zHash) ) zHash[j] = 0; + zHash[sizeof(zHash)-1] = 0; + if( bDebug ){ + printf("%s %s\n", zFile, zHash); + } + if( access(zFile, R_OK)!=0 ){ + xErr(&nErr, zVers, &zFile[nDir]); + continue; + } + if( strlen(zHash)==40 ){ + sha1sum_file(zFile, zCk); + if( strcmp(zHash, zCk)!=0 ){ + xErr(&nErr, zVers, &zFile[nDir]); + } + }else if( strlen(zHash)==64 ){ + sha3sum_file(zFile, zCk); + if( strcmp(zHash, zCk)!=0 ){ + xErr(&nErr, zVers, &zFile[nDir]); + } + }else{ + bSeenManifestErr = 1; + xErr(&nErr, zVers, &zFile[nDir]); + } + } + fclose(in); + in = 0; + if( bSeenManifestErr ) xErr(&nErr, zVers, "manifest"); + memcpy(&zFile[nDir], "manifest.uuid", 14); + if( access(zFile, R_OK)!=0 + || (in = fopen(zFile,"rb"))==0 + || fgets(zLine, sizeof(zLine), in)==0 + || strlen(zLine)!=65 + || zLine[64]!='\n' + || memcmp(zLine, zVers, 64)!=0 + ){ + xErr(&nErr, zVers, &zFile[nDir]); + } + if( in ) fclose(in); + + if( bNonHuman ){ + if( nErr ) return 0; + printf("%s\n", zVers); + }else{ + if( nErr ) return nErr; + printf("OK %.25s\n", zVers); + } + return 0; +} diff --git a/tool/srcck1.c b/tool/srcck1.c new file mode 100644 index 0000000..20084ac --- /dev/null +++ b/tool/srcck1.c @@ -0,0 +1,158 @@ +/* +** The program does some simple static analysis of the sqlite3.c source +** file looking for mistakes. +** +** Usage: +** +** ./srcck1 sqlite3.c +** +** This program looks for instances of assert(), ALWAYS(), NEVER() or +** testcase() that contain side-effects and reports errors if any such +** instances are found. +** +** The aim of this utility is to prevent recurrences of errors such +** as the one fixed at: +** +** https://www.sqlite.org/src/info/a2952231ac7abe16 +** +** Note that another similar error was found by this utility when it was +** first written. That other error was fixed by the same check-in that +** committed the first version of this utility program. +*/ +#include <stdlib.h> +#include <ctype.h> +#include <stdio.h> +#include <string.h> + +/* Read the complete text of a file into memory. Return a pointer to +** the result. Panic if unable to read the file or allocate memory. +*/ +static char *readFile(const char *zFilename){ + FILE *in; + char *z; + long n; + size_t got; + + in = fopen(zFilename, "rb"); + if( in==0 ){ + fprintf(stderr, "unable to open '%s' for reading\n", zFilename); + exit(1); + } + fseek(in, 0, SEEK_END); + n = ftell(in); + rewind(in); + z = malloc( n+1 ); + if( z==0 ){ + fprintf(stderr, "cannot allocate %d bytes to store '%s'\n", + (int)(n+1), zFilename); + exit(1); + } + got = fread(z, 1, n, in); + fclose(in); + if( got!=(size_t)n ){ + fprintf(stderr, "only read %d of %d bytes from '%s'\n", + (int)got, (int)n, zFilename); + exit(1); + } + z[n] = 0; + return z; +} + +/* Check the C code in the argument to see if it might have +** side effects. The only accurate way to know this is to do a full +** parse of the C code, which this routine does not do. This routine +** uses a simple heuristic of looking for: +** +** * '=' not immediately after '>', '<', '!', or '='. +** * '++' +** * '--' +** +** If the code contains the phrase "side-effects-ok" is inside a +** comment, then always return false. This is used to disable checking +** for assert()s with deliberate side-effects, such as used by +** SQLITE_TESTCTRL_ASSERT - a facility that allows applications to +** determine at runtime whether or not assert()s are enabled. +** Obviously, that determination cannot be made unless the assert() +** has some side-effect. +** +** Return true if a side effect is seen. Return false if not. +*/ +static int hasSideEffect(const char *z, unsigned int n){ + unsigned int i; + for(i=0; i<n; i++){ + if( z[i]=='/' && strncmp(&z[i], "/*side-effects-ok*/", 19)==0 ) return 0; + if( z[i]=='=' && i>0 && z[i-1]!='=' && z[i-1]!='>' + && z[i-1]!='<' && z[i-1]!='!' && z[i+1]!='=' ) return 1; + if( z[i]=='+' && z[i+1]=='+' ) return 1; + if( z[i]=='-' && z[i+1]=='-' ) return 1; + } + return 0; +} + +/* Return the number of bytes in string z[] prior to the first unmatched ')' +** character. +*/ +static unsigned int findCloseParen(const char *z){ + unsigned int nOpen = 0; + unsigned i; + for(i=0; z[i]; i++){ + if( z[i]=='(' ) nOpen++; + if( z[i]==')' ){ + if( nOpen==0 ) break; + nOpen--; + } + } + return i; +} + +/* Search for instances of assert(...), ALWAYS(...), NEVER(...), and/or +** testcase(...) where the argument contains side effects. +** +** Print error messages whenever a side effect is found. Return the number +** of problems seen. +*/ +static unsigned int findAllSideEffects(const char *z){ + unsigned int lineno = 1; /* Line number */ + unsigned int i; + unsigned int nErr = 0; + char c, prevC = 0; + for(i=0; (c = z[i])!=0; prevC=c, i++){ + if( c=='\n' ){ lineno++; continue; } + if( isalpha(c) && !isalpha(prevC) ){ + if( strncmp(&z[i],"assert(",7)==0 + || strncmp(&z[i],"ALWAYS(",7)==0 + || strncmp(&z[i],"NEVER(",6)==0 + || strncmp(&z[i],"testcase(",9)==0 + ){ + unsigned int n; + const char *z2 = &z[i+5]; + while( z2[0]!='(' ){ z2++; } + z2++; + n = findCloseParen(z2); + if( hasSideEffect(z2, n) ){ + nErr++; + fprintf(stderr, "side-effect line %u: %.*s\n", lineno, + (int)(&z2[n+1] - &z[i]), &z[i]); + } + } + } + } + return nErr; +} + +int main(int argc, char **argv){ + char *z; + unsigned int nErr = 0; + if( argc!=2 ){ + fprintf(stderr, "Usage: %s FILENAME\n", argv[0]); + return 1; + } + z = readFile(argv[1]); + nErr = findAllSideEffects(z); + free(z); + if( nErr ){ + fprintf(stderr, "Found %u undesirable side-effects\n", nErr); + return 1; + } + return 0; +} diff --git a/tool/srctree-check.tcl b/tool/srctree-check.tcl new file mode 100644 index 0000000..51226cd --- /dev/null +++ b/tool/srctree-check.tcl @@ -0,0 +1,103 @@ +#!/usr/bin/tclsh +# +# Run this script from the top of the source tree in order to confirm that +# various aspects of the source tree are up-to-date. Items checked include: +# +# * Makefile.msc and autoconf/Makefile.msc agree +# * src/ctime.tcl is consistent with tool/mkctimec.tcl +# * VERSION agrees with autoconf/tea/configure.ac +# * src/pragma.h agrees with tool/mkpragmatab.tcl +# +# Other tests might be added later. +# +# Error messages are printed and the process exists non-zero if problems +# are found. If everything is ok, no output is generated and the process +# exits with 0. +# + +# Read an entire file. +# +proc readfile {filename} { + set fd [open $filename rb] + set txt [read $fd] + close $fd + return $txt +} + +# Find the root of the tree. +# +set ROOT [file dir [file dir [file normalize $argv0]]] + +# Name of the TCL interpreter +# +set TCLSH [info nameofexe] + +# Number of errors seen. +# +set NERR 0 + +######################### configure ########################################### + +set conf [readfile $ROOT/configure] +set vers [readfile $ROOT/VERSION] +if {[string first $vers $conf]<=0} { + puts "ERROR: ./configure does not agree with ./VERSION" + puts "...... Fix: run autoconf" + incr NERR +} +unset conf + +######################### autoconf/tea/configure.ac ########################### + +set confac [readfile $ROOT/autoconf/tea/configure.ac] +set vers [readfile $ROOT/VERSION] +set pattern {AC_INIT([sqlite],[} +append pattern [string trim $vers] +append pattern {])} +if {[string first $pattern $confac]<=0} { + puts "ERROR: ./autoconf/tea/configure.ac does not agree with ./VERSION" + puts "...... Fix: manually edit ./autoconf/tea/configure.ac and put the" + puts "...... correct version number in AC_INIT()" + incr NERR +} +unset confac + +######################### autoconf/Makefile.msc ############################### + +set f1 [readfile $ROOT/autoconf/Makefile.msc] +exec $TCLSH $ROOT/tool/mkmsvcmin.tcl $ROOT/Makefile.msc tmp1.txt +set f2 [readfile tmp1.txt] +file delete tmp1.txt +if {$f1 != $f2} { + puts "ERROR: ./autoconf/Makefile.msc does not agree with ./Makefile.msc" + puts "...... Fix: tclsh tool/mkmsvcmin.tcl" + incr NERR +} + +######################### src/pragma.h ######################################## + +set f1 [readfile $ROOT/src/pragma.h] +exec $TCLSH $ROOT/tool/mkpragmatab.tcl tmp2.txt +set f2 [readfile tmp2.txt] +file delete tmp2.txt +if {$f1 != $f2} { + puts "ERROR: ./src/pragma.h does not agree with ./tool/mkpragmatab.tcl" + puts "...... Fix: tclsh tool/mkpragmatab.tcl" + incr NERR +} + +######################### src/ctime.c ######################################## + +set f1 [readfile $ROOT/src/ctime.c] +exec $TCLSH $ROOT/tool/mkctimec.tcl tmp3.txt +set f2 [readfile tmp3.txt] +file delete tmp3.txt +if {$f1 != $f2} { + puts "ERROR: ./src/ctime.c does not agree with ./tool/mkctimec.tcl" + puts "..... Fix: tclsh tool/mkctimec.tcl" + incr NERR +} + +# If any errors are seen, exit 1 so that the build will fail. +# +if {$NERR>0} {exit 1} diff --git a/tool/stack_usage.tcl b/tool/stack_usage.tcl new file mode 100644 index 0000000..b3574f0 --- /dev/null +++ b/tool/stack_usage.tcl @@ -0,0 +1,98 @@ +#!/usr/bin/tclsh +# +# Parse the output of +# +# objdump -d sqlite3.o +# +# for x64 and generate a report showing: +# +# (1) Stack used by each function +# (2) Recursion paths and their aggregate stack depth +# +set getStack 0 +while {![eof stdin]} { + set line [gets stdin] + if {[regexp {^[0-9a-f]+ <([^>]+)>:\s*$} $line all procname]} { + set curfunc $procname + set root($curfunc) 1 + set calls($curfunc) {} + set calledby($curfunc) {} + set recursive($curfunc) {} + set stkdepth($curfunc) 0 + set getStack 1 + continue + } + if {[regexp {callq? +[0-9a-z]+ <([^>]+)>} $line all other]} { + set key [list $curfunc $other] + set callpair($key) 1 + unset -nocomplain root($curfunc) + continue + } + if {[regexp {sub +\$(0x[0-9a-z]+),%[er]sp} $line all xdepth]} { + if {$getStack} { + scan $xdepth %x depth + set stkdepth($curfunc) $depth + set getStack 0 + } + continue + } +} + +puts "****************** Stack Usage By Function ********************" +set sdlist {} +foreach f [array names stkdepth] { + lappend sdlist [list $stkdepth($f) $f] +} +foreach sd [lsort -integer -decr -index 0 $sdlist] { + foreach {depth fname} $sd break + puts [format {%6d %s} $depth $fname] +} + +puts "****************** Stack Usage By Recursion *******************" +foreach key [array names callpair] { + foreach {from to} $key break + lappend calls($from) $to + # lappend calledby($to) $from +} +proc all_descendents {root} { + global calls recursive + set todo($root) $root + set go 1 + while {$go} { + set go 0 + foreach f [array names todo] { + set path $todo($f) + unset todo($f) + if {![info exists calls($f)]} continue + foreach x $calls($f) { + if {$x==$root} { + lappend recursive($root) [concat $path $root] + } elseif {![info exists d($x)]} { + set go 1 + set todo($x) [concat $path $x] + set d($x) 1 + } + } + } + } + return [array names d] +} +set pathlist {} +foreach f [array names recursive] { + all_descendents $f + foreach m $recursive($f) { + set depth 0 + foreach b [lrange $m 0 end-1] { + set depth [expr {$depth+$stkdepth($b)}] + } + lappend pathlist [list $depth $m] + } +} +foreach path [lsort -integer -decr -index 0 $pathlist] { + foreach {depth m} $path break + set first [lindex $m 0] + puts [format {%6d %s %d} $depth $first $stkdepth($first)] + foreach b [lrange $m 1 end] { + puts " $b $stkdepth($b)" + } +} diff --git a/tool/stripccomments.c b/tool/stripccomments.c new file mode 100644 index 0000000..53933c0 --- /dev/null +++ b/tool/stripccomments.c @@ -0,0 +1,228 @@ +/** + Strips C- and C++-style comments from stdin, sending the results to + stdout. It assumes that its input is legal C-like code, and does + only little error handling. + + It treats string literals as anything starting and ending with + matching double OR single quotes OR backticks (for use with + scripting languages which use those). It assumes that a quote + character within a string which uses the same quote type is escaped + by a backslash. It should not be used on any code which might + contain C/C++ comments inside heredocs, and similar constructs, as + it will strip those out. + + Usage: $0 [--keep-first|-k] < input > output + + The --keep-first (-k) flag tells it to retain the first comment in the + input stream (which is often a license or attribution block). It + may be given repeatedly, each one incrementing the number of + retained comments by one. + + License: Public Domain + Author: Stephan Beal (stephan@wanderinghorse.net) +*/ +#include <stdio.h> +#include <assert.h> +#include <string.h> + +#if 1 +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:\t",__FILE__,__LINE__); \ + printf pfexp; \ + } while(0) +#else +#define MARKER(exp) if(0) printf +#endif + +struct { + FILE * input; + FILE * output; + int rc; + int keepFirst; +} App = { + 0/*input*/, + 0/*output*/, + 0/*rc*/, + 0/*keepFirst*/ +}; + +void do_it_all(void){ + enum states { + S_NONE = 0 /* not in comment */, + S_SLASH1 = 1 /* slash - possibly comment prefix */, + S_CPP = 2 /* in C++ comment */, + S_C = 3 /* in C comment */ + }; + int ch, prev = EOF; + FILE * out = App.output; + int const slash = '/'; + int const star = '*'; + int line = 1; + int col = 0; + enum states state = S_NONE /* current state */; + int elide = 0 /* true if currently eliding output */; + int state3Col = -99 + /* huge kludge for odd corner case: */ + /*/ <--- here. state3Col marks the source column in which a C-style + comment starts, so that it can tell if star-slash inside a + C-style comment is the end of the comment or is the weird corner + case marked at the start of _this_ comment block. */; + for( ; EOF != (ch = fgetc(App.input)); prev = ch, + ++col){ + switch(state){ + case S_NONE: + if('\''==ch || '"'==ch || '`'==ch){ + /* Read string literal... + needed to properly catch comments in strings. */ + int const quote = ch, + startLine = line, startCol = col; + int ch2, escaped = 0, endOfString = 0; + fputc(ch, out); + for( ++col; !endOfString && EOF != (ch2 = fgetc(App.input)); + ++col ){ + switch(ch2){ + case '\\': escaped = !escaped; + break; + case '`': + case '\'': + case '"': + if(!escaped && quote == ch2) endOfString = 1; + escaped = 0; + break; + default: + escaped = 0; + break; + } + if('\n'==ch2){ + ++line; + col = 0; + } + fputc(ch2, out); + } + if(EOF == ch2){ + fprintf(stderr, "Unexpected EOF while reading %s literal " + "on line %d column %d.\n", + ('\''==ch) ? "char" : "string", + startLine, startCol); + App.rc = 1; + return; + } + break; + } + else if(slash == ch){ + /* MARKER(("state 0 ==> 1 @ %d:%d\n", line, col)); */ + state = S_SLASH1; + break; + } + fputc(ch, out); + break; + case S_SLASH1: /* 1 slash */ + /* MARKER(("SLASH1 @ %d:%d App.keepFirst=%d\n", + line, col, App.keepFirst)); */ + switch(ch){ + case '*': + /* Enter C comment */ + if(App.keepFirst>0){ + elide = 0; + --App.keepFirst; + }else{ + elide = 1; + } + /*MARKER(("state 1 ==> 3 @ %d:%d\n", line, col));*/ + state = S_C; + state3Col = col-1; + if(!elide){ + fputc(prev, out); + fputc(ch, out); + } + break; + case '/': + /* Enter C++ comment */ + if(App.keepFirst>0){ + elide = 0; + --App.keepFirst; + }else{ + elide = 1; + } + /*MARKER(("state 1 ==> 2 @ %d:%d\n", line, col));*/ + state = S_CPP; + if(!elide){ + fputc(prev, out); + fputc(ch, out); + } + break; + default: + /* It wasn't a comment after all. */ + state = S_NONE; + if(!elide){ + fputc(prev, out); + fputc(ch, out); + } + } + break; + case S_CPP: /* C++ comment */ + if('\n' == ch){ + /* MARKER(("state 2 ==> 0 @ %d:%d\n", line, col)); */ + state = S_NONE; + elide = 0; + } + if(!elide){ + fputc(ch, out); + } + break; + case S_C: /* C comment */ + if(!elide){ + fputc(ch, out); + } + if(slash == ch){ + if(star == prev){ + /* MARKER(("state 3 ==> 0 @ %d:%d\n", line, col)); */ + /* Corner case which breaks this: */ + /*/ <-- slash there */ + /* That shows up twice in a piece of 3rd-party + code i use. */ + /* And thus state3Col was introduced :/ */ + if(col!=state3Col+2){ + state = S_NONE; + elide = 0; + state3Col = -99; + } + } + } + break; + default: + assert(!"impossible!"); + break; + } + if('\n' == ch){ + ++line; + col = 0; + state3Col = -99; + } + } +} + +static void usage(char const *zAppName){ + fprintf(stderr, "Strips C- and C++-style comments from stdin and sends " + "the results to stdout.\n"); + fprintf(stderr, "Usage: %s [--keep-first|-k] < input > output\n", zAppName); +} + +int main( int argc, char const * const * argv ){ + int i; + for(i = 1; i < argc; ++i){ + char const * zArg = argv[i]; + while( '-'==*zArg ) ++zArg; + if( 0==strcmp(zArg,"k") + || 0==strcmp(zArg,"keep-first") ){ + ++App.keepFirst; + }else{ + usage(argv[0]); + return 1; + } + } + App.input = stdin; + App.output = stdout; + do_it_all(); + return App.rc ? 1 : 0; +} diff --git a/tool/symbols-mingw.sh b/tool/symbols-mingw.sh new file mode 100644 index 0000000..bf93eec --- /dev/null +++ b/tool/symbols-mingw.sh @@ -0,0 +1,33 @@ +#!/bin/sh +# +# Run this script in a directory that contains a valid SQLite makefile in +# order to verify that unintentionally exported symbols. +# +make sqlite3.c + +echo '****** Exported symbols from a build including RTREE && FTS4 ******' +gcc -c -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE \ + -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_STAT3 \ + -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_ENABLE_UNLOCK_NOTIFY \ + -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_ATOMIC_WRITE \ + sqlite3.c +nm sqlite3.o | grep " [TD] " + +echo '****** Surplus symbols from a build including RTREE & FTS4 ******' +nm sqlite3.o | grep " [TD] " | grep -v " .*sqlite3_" + +echo '****** Dependencies of the core. No extensions. No OS interface *******' +gcc -c -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_STAT3 \ + -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_ENABLE_UNLOCK_NOTIFY \ + -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_ATOMIC_WRITE \ + -DSQLITE_OS_OTHER -DSQLITE_THREADSAFE=0 \ + sqlite3.c +nm sqlite3.o | grep " U " + +echo '****** Dependencies including RTREE & FTS4 *******' +gcc -c -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE \ + -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_STAT3 \ + -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_ENABLE_UNLOCK_NOTIFY \ + -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_ATOMIC_WRITE \ + sqlite3.c +nm sqlite3.o | grep " U " diff --git a/tool/symbols.sh b/tool/symbols.sh new file mode 100644 index 0000000..67d578e --- /dev/null +++ b/tool/symbols.sh @@ -0,0 +1,36 @@ +#!/bin/sh +# +# Run this script in a directory that contains a valid SQLite makefile in +# order to verify that unintentionally exported symbols. +# +make sqlite3.c + +echo '****** Exported symbols from a build including RTREE, FTS4 & FTS5 ******' +gcc -c -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE \ + -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_STAT3 \ + -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_ENABLE_UNLOCK_NOTIFY \ + -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_ATOMIC_WRITE \ + -DSQLITE_ENABLE_PREUPDATE_HOOK -DSQLITE_ENABLE_SESSION \ + -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_GEOPOLY \ + sqlite3.c +nm sqlite3.o | grep ' [TD] ' | sort -k 3 + +echo '****** Surplus symbols from a build including RTREE, FTS4 & FTS5 ******' +nm sqlite3.o | grep ' [TD] ' | + egrep -v ' .*sqlite3(session|rebaser|changeset|changegroup)?_' + +echo '****** Dependencies of the core. No extensions. No OS interface *******' +gcc -c -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_STAT3 \ + -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_ENABLE_UNLOCK_NOTIFY \ + -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_ATOMIC_WRITE \ + -DSQLITE_OS_OTHER -DSQLITE_THREADSAFE=0 \ + sqlite3.c +nm sqlite3.o | grep ' U ' | sort -k 3 + +echo '****** Dependencies including RTREE & FTS4 *******' +gcc -c -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE \ + -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_STAT3 \ + -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_ENABLE_UNLOCK_NOTIFY \ + -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_ATOMIC_WRITE \ + sqlite3.c +nm sqlite3.o | grep ' U ' | sort -k 3 diff --git a/tool/varint.c b/tool/varint.c new file mode 100644 index 0000000..f4a5111 --- /dev/null +++ b/tool/varint.c @@ -0,0 +1,123 @@ +/* +** A utility program to translate SQLite varints into decimal and decimal +** integers into varints. +*/ +#include <stdio.h> +#include <string.h> +#include <stdlib.h> + +#if defined(_MSC_VER) || defined(__BORLANDC__) + typedef __int64 i64; + typedef unsigned __int64 u64; +#else + typedef long long int i64; + typedef unsigned long long int u64; +#endif + +static int hexValue(char c){ + if( c>='0' && c<='9' ) return c - '0'; + if( c>='a' && c<='f' ) return c - 'a' + 10; + if( c>='A' && c<='F' ) return c - 'A' + 10; + return -1; +} + +static char toHex(unsigned char c){ + return "0123456789abcdef"[c&0xf]; +} + +static int putVarint(unsigned char *p, u64 v){ + int i, j, n; + unsigned char buf[10]; + if( v & (((u64)0xff000000)<<32) ){ + p[8] = (unsigned char)v; + v >>= 8; + for(i=7; i>=0; i--){ + p[i] = (unsigned char)((v & 0x7f) | 0x80); + v >>= 7; + } + return 9; + } + n = 0; + do{ + buf[n++] = (unsigned char)((v & 0x7f) | 0x80); + v >>= 7; + }while( v!=0 ); + buf[0] &= 0x7f; + for(i=0, j=n-1; j>=0; j--, i++){ + p[i] = buf[j]; + } + return n; +} + + +int main(int argc, char **argv){ + int i; + u64 x; + u64 uX = 0; + i64 iX; + int n; + unsigned char zHex[20]; + + if( argc==1 ){ + fprintf(stderr, + "Usage:\n" + " %s HH HH HH ... Convert varint to decimal\n" + " %s DDDDD Convert decimal to varint\n" + " Add '+' or '-' before DDDDD to disambiguate.\n", + argv[0], argv[0]); + exit(1); + } + if( argc>2 + || (strlen(argv[1])==2 && hexValue(argv[1][0])>=0 && hexValue(argv[1][1])>=0) + ){ + /* Hex to decimal */ + for(i=1; i<argc && i<9; i++){ + if( strlen(argv[i])!=2 ){ + fprintf(stderr, "Not a hex byte: %s\n", argv[i]); + exit(1); + } + x = (hexValue(argv[i][0])<<4) + hexValue(argv[i][1]); + uX = (uX<<7) + (x&0x7f); + if( (x&0x80)==0 ) break; + } + if( i==9 && i<argc ){ + if( strlen(argv[i])!=2 ){ + fprintf(stderr, "Not a hex byte: %s\n", argv[i]); + exit(1); + } + x = (hexValue(argv[i][0])<<4) + hexValue(argv[i][1]); + uX = (uX<<8) + x; + } + i++; + if( i<argc ){ + fprintf(stderr, "Extra arguments: %s...\n", argv[i]); + exit(1); + } + }else{ + char *z = argv[1]; + int sign = 1; + if( z[0]=='+' ) z++; + else if( z[0]=='-' ){ z++; sign = -1; } + uX = 0; + while( z[0] ){ + if( z[0]<'0' || z[0]>'9' ){ + fprintf(stderr, "Not a decimal number: %s", argv[1]); + exit(1); + } + uX = uX*10 + z[0] - '0'; + z++; + } + if( sign<0 ){ + memcpy(&iX, &uX, 8); + iX = -iX; + memcpy(&uX, &iX, 8); + } + } + n = putVarint(zHex, uX); + printf("%lld =", (i64)uX); + for(i=0; i<n; i++){ + printf(" %c%c", toHex(zHex[i]>>4), toHex(zHex[i]&0x0f)); + } + printf("\n"); + return 0; +} diff --git a/tool/vdbe-compress.tcl b/tool/vdbe-compress.tcl new file mode 100644 index 0000000..5e63c96 --- /dev/null +++ b/tool/vdbe-compress.tcl @@ -0,0 +1,143 @@ +#!/usr/bin/tcl +# +# This script makes modifications to the vdbe.c source file which reduce +# the amount of stack space required by the sqlite3VdbeExec() routine. +# +# The modifications performed by this script are optional. The vdbe.c +# source file will compile correctly with and without the modifications +# performed by this script. And all routines within vdbe.c will compute +# the same result. The modifications made by this script merely help +# the C compiler to generate code for sqlite3VdbeExec() that uses less +# stack space. +# +# Script usage: +# +# mv vdbe.c vdbe.c.template +# tclsh vdbe-compress.tcl $CFLAGS <vdbe.c.template >vdbe.c +# +# Modifications made: +# +# All modifications are within the sqlite3VdbeExec() function. The +# modifications seek to reduce the amount of stack space allocated by +# this routine by moving local variable declarations out of individual +# opcode implementations and into a single large union. The union contains +# a separate structure for each opcode and that structure contains the +# local variables used by that opcode. In this way, the total amount +# of stack space required by sqlite3VdbeExec() is reduced from the +# sum of all local variables to the maximum of the local variable space +# required for any single opcode. +# +# In order to be recognized by this script, local variables must appear +# on the first line after the open curly-brace that begins a new opcode +# implementation. Local variables must not have initializers, though they +# may be commented. +# +# The union definition is inserted in place of a special marker comment +# in the preamble to the sqlite3VdbeExec() implementation. +# +############################################################################# +# +set beforeUnion {} ;# C code before union +set unionDef {} ;# C code of the union +set afterUnion {} ;# C code after the union +set sCtr 0 ;# Context counter + +# If the SQLITE_SMALL_STACK compile-time option is missing, then +# this transformation becomes a no-op. +# +if {![regexp {SQLITE_SMALL_STACK} $argv]} { + while {![eof stdin]} { + puts [gets stdin] + } + exit +} + +# Read program text up to the spot where the union should be +# inserted. +# +while {![eof stdin]} { + set line [gets stdin] + if {[regexp {INSERT STACK UNION HERE} $line]} break + append beforeUnion $line\n +} + +# Process the remaining text. Build up the union definition as we go. +# +set vlist {} +set seenDecl 0 +set namechars {abcefghjklmnopqrstuvwxyz} +set nnc [string length $namechars] +while {![eof stdin]} { + set line [gets stdin] + if {[regexp "^case (OP_\\w+): \173" $line all operator]} { + append afterUnion $line\n + set vlist {} + while {![eof stdin]} { + set line [gets stdin] + if {[regexp {^ +(const )?\w+ \**(\w+)(\[.*\])?;} $line \ + all constKeyword vname notused1]} { + if {!$seenDecl} { + set sname {} + append sname [string index $namechars [expr {$sCtr/$nnc}]] + append sname [string index $namechars [expr {$sCtr%$nnc}]] + incr sCtr + append unionDef " struct ${operator}_stack_vars \173\n" + append afterUnion \ + "#if 0 /* local variables moved into u.$sname */\n" + set seenDecl 1 + } + append unionDef " $line\n" + append afterUnion $line\n + lappend vlist $vname + } elseif {[regexp {^#(if|endif)} $line] && [llength $vlist]>0} { + append unionDef "$line\n" + append afterUnion $line\n + } else { + break + } + } + if {$seenDecl} { + append unionDef " \175 $sname;\n" + append afterUnion "#endif /* local variables moved into u.$sname */\n" + } + set seenDecl 0 + } + if {[regexp "^\175" $line]} { + append afterUnion $line\n + set vlist {} + } elseif {[llength $vlist]>0} { + append line " " + foreach v $vlist { + regsub -all "(\[^a-zA-Z0-9>.\])${v}(\\W)" $line "\\1u.$sname.$v\\2" line + regsub -all "(\[^a-zA-Z0-9>.\])${v}(\\W)" $line "\\1u.$sname.$v\\2" line + + # The expressions above fail to catch instance of variable "abc" in + # expressions like (32>abc). The following expression makes those + # substitutions. + regsub -all "(\[^-\])>${v}(\\W)" $line "\\1>u.$sname.$v\\2" line + } + append afterUnion [string trimright $line]\n + } elseif {$line=="" && [eof stdin]} { + # no-op + } else { + append afterUnion $line\n + } +} + +# Output the resulting text. +# +puts -nonewline $beforeUnion +puts " /********************************************************************" +puts " ** Automatically generated code" +puts " **" +puts " ** The following union is automatically generated by the" +puts " ** vdbe-compress.tcl script. The purpose of this union is to" +puts " ** reduce the amount of stack space required by this function." +puts " ** See comments in the vdbe-compress.tcl script for details." +puts " */" +puts " union vdbeExecUnion \173" +puts -nonewline $unionDef +puts " \175 u;" +puts " /* End automatically generated code" +puts " ********************************************************************/" +puts -nonewline $afterUnion diff --git a/tool/vdbe_profile.tcl b/tool/vdbe_profile.tcl new file mode 100644 index 0000000..b7240e3 --- /dev/null +++ b/tool/vdbe_profile.tcl @@ -0,0 +1,96 @@ +#!/bin/tclsh +# +# SUMMARY: +# Run this script in the same directory as the "vdbe_profile.out" file. +# This script summarizes the results contained in that file. +# +# DETAILS: +# Compile SQLite using the -DVDBE_PROFILE option on Linux. This causes +# performance information about individual VDBE operations to be appended +# to the "vdbe_profile.out" file. After content has been accumulated in +# vdbe_profile.out, run this script to analyze the output and generate a +# report. +# +if {![file readable vdbe_profile.out]} { + error "run this script in the same directory as the vdbe_profile.out file" +} +set in [open vdbe_profile.out r] +set stmt {} +set allstmt {} +while {![eof $in]} { + set line [gets $in] + if {$line==""} continue + if {[regexp {^---- } $line]} { + set stmt [lindex $line 1] + if {[info exists cnt($stmt)]} { + incr cnt($stmt) + set firsttime 0 + } else { + set cnt($stmt) 1 + set sql($stmt) {} + set firsttime 1 + lappend allstmt $stmt + } + continue; + } + if {[regexp {^-- } $line]} { + if {$firsttime} { + append sql($stmt) [string range $line 3 end]\n + } + continue + } + if {![regexp {^ *\d+ *\d+ *\d+ *\d+ ([A-Z].*)} $line all detail]} continue + set c [lindex $line 0] + set t [lindex $line 1] + set addr [lindex $line 3] + set op [lindex $line 4] + if {[info exists opcnt($op)]} { + incr opcnt($op) $c + incr opcycle($op) $t + } else { + set opcnt($op) $c + set opcycle($op) $t + } + if {[info exists stat($stmt,$addr)]} { + foreach {cx tx detail} $stat($stmt,$addr) break + incr cx $c + incr tx $t + set stat($stmt,$addr) [list $cx $tx $detail] + } else { + set stat($stmt,$addr) [list $c $t $detail] + } +} +close $in + +foreach stmt $allstmt { + puts "********************************************************************" + puts [string trim $sql($stmt)] + puts "Execution count: $cnt($stmt)" + set tcx 0 + set ttx 0 + for {set i 0} {[info exists stat($stmt,$i)]} {incr i} { + foreach {cx tx detail} $stat($stmt,$i) break + if {$cx==0} { + set ax 0 + } else { + set ax [expr {$tx/$cx}] + } + puts [format {%8d %12d %12d %4d %s} $cx $tx $ax $i $detail] + incr tcx $cx + incr ttx $tx + } + set tax [expr {$tcx>0?$ttx/$tcx:0}] + puts [format {%8d %12d %12d TOTAL} $tcx $ttx $tax] +} +puts "********************************************************************" +puts "OPCODES:" +foreach op [lsort [array names opcnt]] { + set cx $opcnt($op) + set tx $opcycle($op) + if {$cx==0} { + set ax 0 + } else { + set ax [expr {$tx/$cx}] + } + puts [format {%8d %12d %12d %s} $cx $tx $ax $op] +} diff --git a/tool/version-info.c b/tool/version-info.c new file mode 100644 index 0000000..62fcd63 --- /dev/null +++ b/tool/version-info.c @@ -0,0 +1,106 @@ +/* +** 2022-10-16 +** +** The author disclaims copyright to this source code. In place of a +** legal notice, here is a blessing: +** +** * May you do good and not evil. +** * May you find forgiveness for yourself and forgive others. +** * May you share freely, never taking more than you give. +** +************************************************************************* +** This file simply outputs sqlite3 version information in JSON form, +** intended for embedding in the sqlite3 JS API build. +*/ +#ifdef TEST_VERSION +/*3029003 3039012*/ +#define SQLITE_VERSION "X.Y.Z" +#define SQLITE_VERSION_NUMBER TEST_VERSION +#define SQLITE_SOURCE_ID "dummy" +#else +#include "sqlite3.h" +#endif +#include <stdio.h> +#include <string.h> +static void usage(const char *zAppName){ + puts("Emits version info about the sqlite3 it is built against."); + printf("Usage: %s [--quote] --INFO-FLAG:\n\n", zAppName); + puts(" --version Emit SQLITE_VERSION (3.X.Y)"); + puts(" --version-number Emit SQLITE_VERSION_NUMBER (30XXYYZZ)"); + puts(" --download-version Emit /download.html version number (3XXYYZZ)"); + puts(" --source-id Emit SQLITE_SOURCE_ID"); + puts(" --json Emit all info in JSON form"); + puts("\nThe non-JSON formats may be modified by:\n"); + puts(" --quote Add double quotes around output."); +} + +int main(int argc, char const * const * argv){ + int fJson = 0; + int fVersion = 0; + int fVersionNumber = 0; + int fDlVersion = 0; + int dlVersion = 0; + int fSourceInfo = 0; + int fQuote = 0; + int nFlags = 0; + int i; + + for( i = 1; i < argc; ++i ){ + const char * zArg = argv[i]; + while('-'==*zArg) ++zArg; + if( 0==strcmp("version", zArg) ){ + fVersion = 1; + }else if( 0==strcmp("version-number", zArg) ){ + fVersionNumber = 1; + }else if( 0==strcmp("download-version", zArg) ){ + fDlVersion = 1; + }else if( 0==strcmp("source-id", zArg) ){ + fSourceInfo = 1; + }else if( 0==strcmp("json", zArg) ){ + fJson = 1; + }else if( 0==strcmp("quote", zArg) ){ + fQuote = 1; + --nFlags; + }else{ + printf("Unhandled flag: %s\n", argv[i]); + usage(argv[0]); + return 1; + } + ++nFlags; + } + + if( 0==nFlags ) fJson = 1; + + { + const int v = SQLITE_VERSION_NUMBER; + int ver[4] = {0,0,0,0}; + ver[0] = (v / 1000000) * 1000000; + ver[1] = v % 1000000 / 100 * 1000; + ver[2] = v % 100 * 100; + dlVersion = ver[0] + ver[1] + ver[2] + ver[3]; + } + if( fJson ){ + printf("{\"libVersion\": \"%s\", " + "\"libVersionNumber\": %d, " + "\"sourceId\": \"%s\"," + "\"downloadVersion\": %d}"/*missing newline is intentional*/, + SQLITE_VERSION, + SQLITE_VERSION_NUMBER, + SQLITE_SOURCE_ID, + dlVersion); + }else{ + if(fQuote) printf("%c", '"'); + if( fVersion ){ + printf("%s", SQLITE_VERSION); + }else if( fVersionNumber ){ + printf("%d", SQLITE_VERSION_NUMBER); + }else if( fSourceInfo ){ + printf("%s", SQLITE_SOURCE_ID); + }else if( fDlVersion ){ + printf("%d", dlVersion); + } + if(fQuote) printf("%c", '"'); + puts(""); + } + return 0; +} diff --git a/tool/warnings-clang.sh b/tool/warnings-clang.sh new file mode 100644 index 0000000..6dcc086 --- /dev/null +++ b/tool/warnings-clang.sh @@ -0,0 +1,14 @@ +#/bin/sh +# +# Run this script in a directory with a working makefile to check for +# compiler warnings in SQLite. +# +rm -f sqlite3.c shell.c +make sqlite3.c shell.c +echo '************* FTS4 and RTREE ****************' +scan-build gcc -c -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE \ + -DSQLITE_DEBUG -DSQLITE_ENABLE_STAT3 sqlite3.c 2>&1 | grep -v 'ANALYZE:' +echo '********** ENABLE_STAT3. THREADSAFE=0 *******' +scan-build gcc -c -I. -DSQLITE_ENABLE_STAT3 -DSQLITE_THREADSAFE=0 \ + -DSQLITE_DEBUG \ + sqlite3.c shell.c -ldl 2>&1 | grep -v 'ANALYZE:' diff --git a/tool/warnings.sh b/tool/warnings.sh new file mode 100644 index 0000000..2b962d1 --- /dev/null +++ b/tool/warnings.sh @@ -0,0 +1,65 @@ +#/bin/sh +# +# Run this script in a directory with a working makefile to check for +# compiler warnings in SQLite. +# + +if uname | grep -i openbsd ; then + # Use these for testing on OpenBSD: + WARNING_OPTS=-Wall + WARNING_ANDROID_OPTS=-Wall +else + # Use these for testing on Linux and Mac OSX: + WARNING_OPTS="-Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long" + gccvers=`gcc -v 2>&1 | grep '^gcc version'` + if test "$gccvers" '<' 'gcc version 6' + then + WARNING_ANDROID_OPTS="-Wshadow -Wall -Wextra" + else + WARNING_ANDROID_OPTS="-Wshadow -Wall -Wextra -Wimplicit-fallthrough=0" + fi +fi + +rm -f sqlite3.c +make sqlite3.c +echo '********** No optimizations. Includes FTS4/5, GEOPOLY, JSON1 ***' +echo '********** ' Options: $WARNING_OPTS +gcc -c $WARNING_OPTS -std=c89 \ + -ansi -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_GEOPOLY \ + -DSQLITE_ENABLE_FTS5 \ + sqlite3.c +if test x`uname` = 'xLinux'; then +echo '********** Android configuration ******************************' +echo '********** ' Options: $WARNING_ANDROID_OPTS +gcc -c \ + -DSQLITE_HAVE_ISNAN \ + -DSQLITE_DEFAULT_JOURNAL_SIZE_LIMIT=1048576 \ + -DSQLITE_THREADSAFE=2 \ + -DSQLITE_TEMP_STORE=3 \ + -DSQLITE_POWERSAFE_OVERWRITE=1 \ + -DSQLITE_DEFAULT_FILE_FORMAT=4 \ + -DSQLITE_DEFAULT_AUTOVACUUM=1 \ + -DSQLITE_ENABLE_MEMORY_MANAGEMENT=1 \ + -DSQLITE_ENABLE_FTS3 \ + -DSQLITE_ENABLE_FTS3_BACKWARDS \ + -DSQLITE_ENABLE_FTS4 \ + -DSQLITE_OMIT_BUILTIN_TEST \ + -DSQLITE_OMIT_COMPILEOPTION_DIAGS \ + -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_DEFAULT_FILE_PERMISSIONS=0600 \ + -DSQLITE_ENABLE_ICU \ + -DUSE_PREAD64 \ + $WARNING_ANDROID_OPTS \ + -Os sqlite3.c shell.c +fi +echo '********** No optimizations. ENABLE_STAT4. THREADSAFE=0 *******' +echo '********** ' Options: $WARNING_OPTS +gcc -c $WARNING_OPTS -std=c89 \ + -ansi -DSQLITE_ENABLE_STAT4 -DSQLITE_THREADSAFE=0 \ + sqlite3.c +echo '********** Optimized -O3. Includes FTS4/5, GEOPOLY, JSON1 ******' +echo '********** ' Options: $WARNING_OPTS +gcc -O3 -c $WARNING_OPTS -std=c89 \ + -ansi -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_GEOPOLY \ + -DSQLITE_ENABLE_FTS5 \ + sqlite3.c diff --git a/tool/win/sqlite.vsix b/tool/win/sqlite.vsix Binary files differnew file mode 100644 index 0000000..1450011 --- /dev/null +++ b/tool/win/sqlite.vsix |