From 347c164c35eddab388009470e6848cb361ac93f8 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sat, 18 May 2024 15:22:53 +0200 Subject: Merging upstream version 1:10.11.8. Signed-off-by: Daniel Baumann --- extra/CMakeLists.txt | 2 +- extra/mariabackup/CMakeLists.txt | 15 +- extra/mariabackup/aria_backup_client.cc | 1016 + extra/mariabackup/aria_backup_client.h | 38 + extra/mariabackup/backup_copy.cc | 495 +- extra/mariabackup/backup_copy.h | 32 +- extra/mariabackup/backup_debug.h | 21 +- extra/mariabackup/backup_mysql.cc | 356 +- extra/mariabackup/backup_mysql.h | 24 +- extra/mariabackup/changed_page_bitmap.cc | 1040 - extra/mariabackup/changed_page_bitmap.h | 85 - extra/mariabackup/common.h | 3 +- extra/mariabackup/common_engine.cc | 512 + extra/mariabackup/common_engine.h | 39 + extra/mariabackup/datasink.cc | 28 +- extra/mariabackup/datasink.h | 17 +- extra/mariabackup/ddl_log.cc | 553 + extra/mariabackup/ddl_log.h | 15 + extra/mariabackup/ds_buffer.cc | 9 +- extra/mariabackup/ds_compress.cc | 9 +- extra/mariabackup/ds_local.cc | 105 +- extra/mariabackup/ds_stdout.cc | 8 +- extra/mariabackup/ds_tmpfile.cc | 8 +- extra/mariabackup/ds_xbstream.cc | 61 +- extra/mariabackup/encryption_plugin.cc | 249 + extra/mariabackup/encryption_plugin.h | 7 + extra/mariabackup/fil_cur.cc | 19 +- extra/mariabackup/fil_cur.h | 1 + extra/mariabackup/innobackupex.cc | 78 +- extra/mariabackup/read_filt.cc | 142 +- extra/mariabackup/read_filt.h | 29 +- extra/mariabackup/thread_pool.cc | 50 + extra/mariabackup/thread_pool.h | 62 + extra/mariabackup/write_filt.cc | 12 + extra/mariabackup/wsrep.cc | 48 +- extra/mariabackup/xb_plugin.cc | 229 - extra/mariabackup/xb_plugin.h | 5 - extra/mariabackup/xbstream.cc | 64 +- extra/mariabackup/xbstream.h | 19 +- extra/mariabackup/xbstream_read.cc | 107 +- extra/mariabackup/xbstream_write.cc | 152 +- extra/mariabackup/xtrabackup.cc | 1005 +- extra/mariabackup/xtrabackup.h | 57 +- extra/wolfssl/CMakeLists.txt | 184 +- extra/wolfssl/user_settings.h.in | 18 +- extra/wolfssl/wolfssl/.github/workflows/async.yml | 2 +- extra/wolfssl/wolfssl/.github/workflows/curl.yml | 8 +- .../wolfssl/.github/workflows/docker-Espressif.yml | 6 +- .../wolfssl/.github/workflows/docker-OpenWrt.yml | 16 +- .../wolfssl/wolfssl/.github/workflows/haproxy.yml | 4 +- extra/wolfssl/wolfssl/.github/workflows/hitch.yml | 10 +- extra/wolfssl/wolfssl/.github/workflows/hostap.yml | 14 +- extra/wolfssl/wolfssl/.github/workflows/krb5.yml | 20 +- .../wolfssl/wolfssl/.github/workflows/libssh2.yml | 58 + extra/wolfssl/wolfssl/.github/workflows/main.yml | 5 + .../wolfssl/.github/workflows/memcached.yml | 10 +- .../wolfssl/.github/workflows/multi-arch.yml | 2 +- .../wolfssl/.github/workflows/multi-compiler.yml | 2 +- extra/wolfssl/wolfssl/.github/workflows/nginx.yml | 18 +- .../wolfssl/wolfssl/.github/workflows/openssh.yml | 68 + .../wolfssl/wolfssl/.github/workflows/openvpn.yml | 14 +- .../wolfssl/wolfssl/.github/workflows/os-check.yml | 7 +- .../wolfssl/.github/workflows/packaging.yml | 2 +- .../wolfssl/wolfssl/.github/workflows/stunnel.yml | 8 +- extra/wolfssl/wolfssl/.github/workflows/zephyr.yml | 8 +- extra/wolfssl/wolfssl/.gitignore | 40 +- extra/wolfssl/wolfssl/CMakeLists.txt | 227 +- extra/wolfssl/wolfssl/ChangeLog.md | 94 + extra/wolfssl/wolfssl/Docker/Dockerfile | 5 +- .../wolfssl/Docker/Dockerfile.cross-compiler | 2 +- extra/wolfssl/wolfssl/Docker/yocto/Dockerfile | 27 + extra/wolfssl/wolfssl/Docker/yocto/buildAndPush.sh | 27 + .../wolfssl/IDE/ARDUINO/Arduino_README_prepend.md | 13 + extra/wolfssl/wolfssl/IDE/ARDUINO/README.md | 185 +- extra/wolfssl/wolfssl/IDE/ARDUINO/include.am | 9 + extra/wolfssl/wolfssl/IDE/ARDUINO/keywords.txt | 21 + .../IDE/ARDUINO/library.properties.template | 9 + .../wolfssl/wolfssl/IDE/ARDUINO/sketches/README.md | 12 + .../IDE/ARDUINO/sketches/wolfssl_client/README.md | 22 + .../sketches/wolfssl_client/wolfssl_client.ino | 928 +- .../IDE/ARDUINO/sketches/wolfssl_server/README.md | 134 + .../sketches/wolfssl_server/wolfssl_server.ino | 913 +- .../IDE/ARDUINO/sketches/wolfssl_version/README.md | 3 + .../sketches/wolfssl_version/wolfssl_version.ino | 24 + .../wolfssl/wolfssl/IDE/ARDUINO/wolfssl-arduino.sh | 347 +- extra/wolfssl/wolfssl/IDE/ARDUINO/wolfssl.h | 39 + .../template/components/wolfssl/CMakeLists.txt | 6 +- .../components/wolfssl/CMakeLists.txt | 6 +- .../ESP-IDF/examples/wolfssl_benchmark/main/main.c | 33 +- .../ESP-IDF/examples/wolfssl_client/Makefile | 128 +- .../ESP-IDF/examples/wolfssl_client/README.md | 122 +- .../VisualGDB/wolfssl_client_IDF_v5_ESP32.sln | 5 + .../components/wolfssl/CMakeLists.txt | 10 +- .../wolfssl_client/components/wolfssl/component.mk | 230 +- .../components/wolfssl/include/user_settings.h | 199 +- .../examples/wolfssl_client/main/client-tls.c | 122 +- .../examples/wolfssl_client/main/component.mk | 6 +- .../wolfssl_client/main/include/client-tls.h | 26 +- .../examples/wolfssl_client/main/include/main.h | 4 +- .../wolfssl_client/main/include/time_helper.h | 15 +- .../wolfssl_client/main/include/wifi_connect.h | 58 +- .../ESP-IDF/examples/wolfssl_client/main/main.c | 114 +- .../examples/wolfssl_client/main/time_helper.c | 162 +- .../examples/wolfssl_client/main/wifi_connect.c | 136 +- .../examples/wolfssl_client/sdkconfig.defaults | 29 +- .../wolfssl_client/wolfssl_client_ESP8266.vgdbproj | 292 + .../components/wolfssl/CMakeLists.txt | 6 +- .../wolfssl_test/components/wolfssl/CMakeLists.txt | 6 +- .../ESP-IDF/examples/wolfssl_test/main/main.c | 20 +- extra/wolfssl/wolfssl/IDE/Espressif/README.md | 43 +- extra/wolfssl/wolfssl/IDE/Espressif/include.am | 1 + .../wolfssl/IDE/IAR-EWARM/Projects/user_settings.h | 52 +- .../settings/wolfcrypt_benchmark_Debug.jlink | 39 - .../wolfcrypt_benchmark.ewt | 2382 -- .../wolfcrypt_lib.ewt | 2400 -- .../settings/wolfcrypt_test_Debug.jlink | 39 - .../wolfcrypt_test.ewt | 2382 -- .../wolfssl/wolfssl/IDE/LINUX-SGX/sgx_t_static.mk | 1 + extra/wolfssl/wolfssl/IDE/MPLABX16/README.md | 62 + extra/wolfssl/wolfssl/IDE/MPLABX16/include.am | 8 + extra/wolfssl/wolfssl/IDE/MPLABX16/main.c | 39 + extra/wolfssl/wolfssl/IDE/MPLABX16/user_settings.h | 414 + .../wolfssl/IDE/MPLABX16/wolfcrypt_test.X/Makefile | 113 + .../wolfcrypt_test.X/nbproject/configurations.xml | 285 + .../MPLABX16/wolfcrypt_test.X/nbproject/include.am | 8 + .../nbproject/private/configurations.xml | 25 + .../wolfcrypt_test.X/nbproject/private/private.xml | 7 + .../wolfcrypt_test.X/nbproject/project.xml | 33 + .../wolfssl/IDE/MPLABX16/wolfssl.X/Makefile | 113 + .../wolfssl.X/nbproject/configurations.xml | 290 + .../IDE/MPLABX16/wolfssl.X/nbproject/include.am | 7 + .../IDE/MPLABX16/wolfssl.X/nbproject/project.xml | 34 + .../wolfssl/IDE/MSVS-2019-AZSPHERE/shared/util.h | 1 + .../Renesas/e2studio/RA6M4/common/user_settings.h | 3 +- .../IDE/Renesas/e2studio/RA6M4/test/.cproject | 4 +- .../wolfssl/IDE/STM32Cube/STM32_Benchmarks.md | 75 + .../wolfssl/wolfssl/IDE/STM32Cube/default_conf.ftl | 47 +- extra/wolfssl/wolfssl/IDE/WIN10/wolfssl-fips.rc | 8 +- extra/wolfssl/wolfssl/IDE/include.am | 3 + extra/wolfssl/wolfssl/IDE/iotsafe/memory-tls.c | 5 +- extra/wolfssl/wolfssl/IDE/iotsafe/user_settings.h | 16 +- extra/wolfssl/wolfssl/IPP/.gitkeep | 0 extra/wolfssl/wolfssl/Makefile.am | 4 - extra/wolfssl/wolfssl/README | 140 +- extra/wolfssl/wolfssl/README.md | 187 +- extra/wolfssl/wolfssl/autogen.sh | 28 - extra/wolfssl/wolfssl/certs/crl/caEcc384Crl.pem | 12 +- extra/wolfssl/wolfssl/certs/crl/caEccCrl.pem | 12 +- extra/wolfssl/wolfssl/certs/crl/cliCrl.pem | 54 +- extra/wolfssl/wolfssl/certs/crl/crl.der | Bin 520 -> 520 bytes extra/wolfssl/wolfssl/certs/crl/crl.pem | 52 +- extra/wolfssl/wolfssl/certs/crl/crl.revoked | 56 +- extra/wolfssl/wolfssl/certs/crl/crl2.der | Bin 520 -> 520 bytes extra/wolfssl/wolfssl/certs/crl/crl2.pem | 102 +- extra/wolfssl/wolfssl/certs/crl/crl_rsapss.pem | 53 + extra/wolfssl/wolfssl/certs/crl/eccCliCRL.pem | 22 +- extra/wolfssl/wolfssl/certs/crl/eccSrvCRL.pem | 22 +- .../certs/crl/extra-crls/ca-int-cert-revoked.pem | 16 +- .../certs/crl/extra-crls/general-server-crl.pem | 16 +- extra/wolfssl/wolfssl/certs/crl/gencrls.sh | 17 + extra/wolfssl/wolfssl/certs/crl/include.am | 3 +- extra/wolfssl/wolfssl/certs/include.am | 1 + extra/wolfssl/wolfssl/certs/renewcerts.sh | 5 + extra/wolfssl/wolfssl/certs/test-stream-sign.p7b | Bin 0 -> 6228 bytes extra/wolfssl/wolfssl/cmake/config.in | 6 + extra/wolfssl/wolfssl/cmake/functions.cmake | 30 +- extra/wolfssl/wolfssl/configure.ac | 465 +- .../doc/dox_comments/header_files-ja/cmac.h | 54 +- .../dox_comments/header_files-ja/doxygen_groups.h | 1 + .../dox_comments/header_files-ja/doxygen_pages.h | 1 + .../wolfssl/doc/dox_comments/header_files-ja/rsa.h | 2 - .../wolfssl/doc/dox_comments/header_files/aes.h | 22 +- .../wolfssl/doc/dox_comments/header_files/cmac.h | 5 +- .../wolfssl/doc/dox_comments/header_files/ecc.h | 8 +- .../wolfssl/doc/dox_comments/header_files/hmac.h | 439 + .../doc/dox_comments/header_files/iotsafe.h | 8 +- .../wolfssl/doc/dox_comments/header_files/rsa.h | 51 +- .../wolfssl/doc/dox_comments/header_files/ssl.h | 208 +- extra/wolfssl/wolfssl/examples/asn1/asn1.c | 4 +- extra/wolfssl/wolfssl/examples/client/client.c | 18 + extra/wolfssl/wolfssl/examples/configs/include.am | 2 + .../examples/configs/user_settings_arduino.h | 486 + .../wolfssl/examples/configs/user_settings_tls12.h | 158 + extra/wolfssl/wolfssl/examples/pem/pem.c | 4 +- extra/wolfssl/wolfssl/examples/server/server.c | 37 +- extra/wolfssl/wolfssl/fips-check.sh | 40 +- extra/wolfssl/wolfssl/lib/dummy | 2 - extra/wolfssl/wolfssl/linuxkm/Kbuild | 21 +- extra/wolfssl/wolfssl/linuxkm/Makefile | 4 + extra/wolfssl/wolfssl/linuxkm/include.am | 3 +- extra/wolfssl/wolfssl/linuxkm/linuxkm_memory.c | 124 +- extra/wolfssl/wolfssl/linuxkm/linuxkm_wc_port.h | 280 +- extra/wolfssl/wolfssl/linuxkm/lkcapi_glue.c | 2739 ++ extra/wolfssl/wolfssl/linuxkm/module_hooks.c | 144 +- extra/wolfssl/wolfssl/pre-commit.sh | 45 - extra/wolfssl/wolfssl/pre-push.sh | 19 - extra/wolfssl/wolfssl/rpm/spec.in | 1 - extra/wolfssl/wolfssl/scripts/makedistsmall.sh | 1 - extra/wolfssl/wolfssl/scripts/openssl.test | 2 +- extra/wolfssl/wolfssl/src/bio.c | 69 +- extra/wolfssl/wolfssl/src/crl.c | 64 +- extra/wolfssl/wolfssl/src/dtls.c | 8 + extra/wolfssl/wolfssl/src/dtls13.c | 1 + extra/wolfssl/wolfssl/src/include.am | 59 +- extra/wolfssl/wolfssl/src/internal.c | 880 +- extra/wolfssl/wolfssl/src/pk.c | 93 +- extra/wolfssl/wolfssl/src/quic.c | 13 + extra/wolfssl/wolfssl/src/sniffer.c | 19 +- extra/wolfssl/wolfssl/src/ssl.c | 852 +- extra/wolfssl/wolfssl/src/ssl_bn.c | 3 +- extra/wolfssl/wolfssl/src/ssl_certman.c | 87 +- extra/wolfssl/wolfssl/src/tls.c | 328 +- extra/wolfssl/wolfssl/src/tls13.c | 934 +- extra/wolfssl/wolfssl/src/wolfio.c | 53 + extra/wolfssl/wolfssl/src/x509.c | 152 +- extra/wolfssl/wolfssl/src/x509_str.c | 10 +- extra/wolfssl/wolfssl/support/wolfssl.pc.in | 1 + extra/wolfssl/wolfssl/tests/api.c | 2731 +- extra/wolfssl/wolfssl/tests/srp.c | 2 + extra/wolfssl/wolfssl/tests/suites.c | 2 +- extra/wolfssl/wolfssl/tests/test-dtls.conf | 14 + extra/wolfssl/wolfssl/tests/test.conf | 12 + extra/wolfssl/wolfssl/tests/unit.h | 3 +- extra/wolfssl/wolfssl/tests/utils.h | 70 +- extra/wolfssl/wolfssl/testsuite/testsuite.c | 4 +- .../wolfssl/wolfcrypt/benchmark/benchmark.c | 2552 +- .../wolfssl/wolfcrypt/benchmark/benchmark.h | 2 +- extra/wolfssl/wolfssl/wolfcrypt/src/aes.c | 251 +- extra/wolfssl/wolfssl/wolfcrypt/src/aes_asm.S | 6 +- .../wolfssl/wolfcrypt/src/aes_gcm_x86_asm.S | 300 +- extra/wolfssl/wolfssl/wolfcrypt/src/asn.c | 1298 +- extra/wolfssl/wolfssl/wolfcrypt/src/chacha.c | 9 +- .../wolfssl/wolfcrypt/src/chacha20_poly1305.c | 52 +- extra/wolfssl/wolfssl/wolfcrypt/src/cmac.c | 162 +- extra/wolfssl/wolfssl/wolfcrypt/src/cpuid.c | 1 + extra/wolfssl/wolfssl/wolfcrypt/src/cryptocb.c | 314 +- extra/wolfssl/wolfssl/wolfcrypt/src/curve25519.c | 8 + extra/wolfssl/wolfssl/wolfcrypt/src/dh.c | 18 +- extra/wolfssl/wolfssl/wolfcrypt/src/dilithium.c | 206 +- extra/wolfssl/wolfssl/wolfcrypt/src/dsa.c | 8 + extra/wolfssl/wolfssl/wolfcrypt/src/ecc.c | 284 +- extra/wolfssl/wolfssl/wolfcrypt/src/eccsi.c | 8 + extra/wolfssl/wolfssl/wolfcrypt/src/ed25519.c | 26 + extra/wolfssl/wolfssl/wolfcrypt/src/ed448.c | 36 +- extra/wolfssl/wolfssl/wolfcrypt/src/error.c | 3 + extra/wolfssl/wolfssl/wolfcrypt/src/evp.c | 92 +- extra/wolfssl/wolfssl/wolfcrypt/src/ext_kyber.c | 76 +- extra/wolfssl/wolfssl/wolfcrypt/src/ext_xmss.c | 64 + extra/wolfssl/wolfssl/wolfcrypt/src/falcon.c | 190 +- .../wolfssl/wolfssl/wolfcrypt/src/fe_x25519_asm.S | 36 +- extra/wolfssl/wolfssl/wolfcrypt/src/ge_448.c | 46 +- extra/wolfssl/wolfssl/wolfcrypt/src/hash.c | 305 +- extra/wolfssl/wolfssl/wolfcrypt/src/hmac.c | 27 +- extra/wolfssl/wolfssl/wolfcrypt/src/include.am | 6 +- extra/wolfssl/wolfssl/wolfcrypt/src/integer.c | 3 + extra/wolfssl/wolfssl/wolfcrypt/src/kdf.c | 121 +- extra/wolfssl/wolfssl/wolfcrypt/src/logging.c | 13 +- extra/wolfssl/wolfssl/wolfcrypt/src/md5.c | 5 +- extra/wolfssl/wolfssl/wolfcrypt/src/memory.c | 57 +- extra/wolfssl/wolfssl/wolfcrypt/src/misc.c | 62 + extra/wolfssl/wolfssl/wolfcrypt/src/pkcs12.c | 2 +- extra/wolfssl/wolfssl/wolfcrypt/src/pkcs7.c | 2181 +- .../wolfcrypt/src/port/Espressif/esp32_mp.c | 10 +- .../wolfcrypt/src/port/Espressif/esp32_sha.c | 36 +- .../wolfcrypt/src/port/Espressif/esp32_util.c | 17 +- .../wolfssl/wolfcrypt/src/port/aria/aria-crypt.c | 5 + .../wolfcrypt/src/port/aria/aria-cryptocb.c | 28 +- .../wolfssl/wolfcrypt/src/port/arm/armv8-aes.c | 76 +- .../wolfssl/wolfcrypt/src/port/arm/armv8-chacha.c | 34 +- .../wolfssl/wolfcrypt/src/port/arm/armv8-sha256.c | 198 +- .../wolfcrypt/src/port/arm/thumb2-aes-asm.S | 29 +- .../wolfcrypt/src/port/arm/thumb2-aes-asm_c.c | 104 +- .../wolfcrypt/src/port/arm/thumb2-sha256-asm_c.c | 5 +- .../wolfcrypt/src/port/arm/thumb2-sha512-asm_c.c | 3 +- .../wolfssl/wolfcrypt/src/port/autosar/README.md | 45 + .../wolfssl/wolfcrypt/src/port/autosar/cryif.c | 103 + .../wolfssl/wolfcrypt/src/port/autosar/crypto.c | 495 + .../wolfssl/wolfcrypt/src/port/autosar/csm.c | 298 + .../wolfssl/wolfcrypt/src/port/autosar/include.am | 23 + .../wolfssl/wolfcrypt/src/port/autosar/test.c | 430 + .../wolfssl/wolfcrypt/src/port/iotsafe/iotsafe.c | 44 +- .../wolfssl/wolfcrypt/src/port/liboqs/liboqs.c | 132 + .../wolfssl/wolfssl/wolfcrypt/src/port/st/stm32.c | 52 +- .../wolfssl/wolfssl/wolfcrypt/src/port/ti/ti-aes.c | 1066 +- .../wolfssl/wolfcrypt/src/port/ti/ti-hash.c | 39 +- .../wolfssl/wolfcrypt/src/port/xilinx/xil-aesgcm.c | 33 +- extra/wolfssl/wolfssl/wolfcrypt/src/random.c | 359 +- extra/wolfssl/wolfssl/wolfcrypt/src/rsa.c | 93 +- extra/wolfssl/wolfssl/wolfcrypt/src/sakke.c | 9 +- extra/wolfssl/wolfssl/wolfcrypt/src/sha.c | 66 +- extra/wolfssl/wolfssl/wolfcrypt/src/sha256.c | 407 +- extra/wolfssl/wolfssl/wolfcrypt/src/sha256_asm.S | 942 +- extra/wolfssl/wolfssl/wolfcrypt/src/sha512.c | 83 +- extra/wolfssl/wolfssl/wolfcrypt/src/siphash.c | 12 +- extra/wolfssl/wolfssl/wolfcrypt/src/sp_arm32.c | 48 +- extra/wolfssl/wolfssl/wolfcrypt/src/sp_arm64.c | 42 +- extra/wolfssl/wolfssl/wolfcrypt/src/sp_armthumb.c | 48 +- extra/wolfssl/wolfssl/wolfcrypt/src/sp_c32.c | 24 +- extra/wolfssl/wolfssl/wolfcrypt/src/sp_c64.c | 24 +- extra/wolfssl/wolfssl/wolfcrypt/src/sp_cortexm.c | 48 +- extra/wolfssl/wolfssl/wolfcrypt/src/sp_dsp32.c | 6 +- extra/wolfssl/wolfssl/wolfcrypt/src/sp_int.c | 59 +- extra/wolfssl/wolfssl/wolfcrypt/src/sp_x86_64.c | 32 +- extra/wolfssl/wolfssl/wolfcrypt/src/sphincs.c | 23 +- extra/wolfssl/wolfssl/wolfcrypt/src/tfm.c | 24 +- extra/wolfssl/wolfssl/wolfcrypt/src/wc_dsp.c | 6 +- extra/wolfssl/wolfssl/wolfcrypt/src/wc_kyber.c | 1247 +- extra/wolfssl/wolfssl/wolfcrypt/src/wc_kyber_asm.S | 27812 ++++++++++++++++++- .../wolfssl/wolfssl/wolfcrypt/src/wc_kyber_poly.c | 3020 +- extra/wolfssl/wolfssl/wolfcrypt/src/wc_lms.c | 2 +- extra/wolfssl/wolfssl/wolfcrypt/src/wc_lms_impl.c | 26 + extra/wolfssl/wolfssl/wolfcrypt/src/wc_port.c | 78 +- extra/wolfssl/wolfssl/wolfcrypt/src/wc_xmss_impl.c | 26 + extra/wolfssl/wolfssl/wolfcrypt/test/test.c | 1393 +- .../wolfssl/wolfcrypt/user-crypto/Makefile.am | 9 - .../wolfssl/wolfcrypt/user-crypto/README.txt | 78 - .../wolfssl/wolfcrypt/user-crypto/autogen.sh | 23 - .../wolfssl/wolfcrypt/user-crypto/configure.ac | 44 - .../wolfssl/wolfcrypt/user-crypto/include.am | 13 - .../wolfcrypt/user-crypto/include/user_rsa.h | 137 - .../wolfssl/wolfcrypt/user-crypto/lib/.gitkeep | 0 .../wolfssl/wolfcrypt/user-crypto/src/rsa.c | 2797 -- extra/wolfssl/wolfssl/wolfssl.rc | Bin 4918 -> 4918 bytes extra/wolfssl/wolfssl/wolfssl/internal.h | 121 +- extra/wolfssl/wolfssl/wolfssl/openssl/bio.h | 1 + extra/wolfssl/wolfssl/wolfssl/openssl/crypto.h | 8 +- extra/wolfssl/wolfssl/wolfssl/openssl/opensslv.h | 9 +- extra/wolfssl/wolfssl/wolfssl/openssl/ssl.h | 3 + extra/wolfssl/wolfssl/wolfssl/quic.h | 11 + extra/wolfssl/wolfssl/wolfssl/ssl.h | 58 +- extra/wolfssl/wolfssl/wolfssl/test.h | 147 +- extra/wolfssl/wolfssl/wolfssl/version.h | 4 +- extra/wolfssl/wolfssl/wolfssl/wolfcrypt/aes.h | 14 +- extra/wolfssl/wolfssl/wolfssl/wolfcrypt/asn.h | 76 +- .../wolfssl/wolfssl/wolfssl/wolfcrypt/asn_public.h | 30 +- extra/wolfssl/wolfssl/wolfssl/wolfcrypt/cmac.h | 14 + extra/wolfssl/wolfssl/wolfssl/wolfcrypt/cpuid.h | 2 + extra/wolfssl/wolfssl/wolfssl/wolfcrypt/cryptocb.h | 106 + .../wolfssl/wolfssl/wolfssl/wolfcrypt/dilithium.h | 41 +- extra/wolfssl/wolfssl/wolfssl/wolfcrypt/ecc.h | 6 +- extra/wolfssl/wolfssl/wolfssl/wolfcrypt/ed25519.h | 2 - extra/wolfssl/wolfssl/wolfssl/wolfcrypt/ed448.h | 2 +- .../wolfssl/wolfssl/wolfcrypt/error-crypt.h | 11 + .../wolfssl/wolfssl/wolfssl/wolfcrypt/ext_kyber.h | 10 + extra/wolfssl/wolfssl/wolfssl/wolfcrypt/falcon.h | 42 +- extra/wolfssl/wolfssl/wolfssl/wolfcrypt/ge_448.h | 2 +- extra/wolfssl/wolfssl/wolfssl/wolfcrypt/hash.h | 33 + extra/wolfssl/wolfssl/wolfssl/wolfcrypt/hmac.h | 8 + extra/wolfssl/wolfssl/wolfssl/wolfcrypt/include.am | 12 + extra/wolfssl/wolfssl/wolfssl/wolfcrypt/kdf.h | 11 + extra/wolfssl/wolfssl/wolfssl/wolfcrypt/kyber.h | 6 +- extra/wolfssl/wolfssl/wolfssl/wolfcrypt/logging.h | 5 + extra/wolfssl/wolfssl/wolfssl/wolfcrypt/memory.h | 47 +- extra/wolfssl/wolfssl/wolfssl/wolfcrypt/misc.h | 3 + extra/wolfssl/wolfssl/wolfssl/wolfcrypt/pkcs7.h | 21 + .../wolfssl/wolfcrypt/port/Espressif/esp32-crypt.h | 38 +- .../wolfssl/wolfssl/wolfcrypt/port/autosar/CryIf.h | 49 + .../wolfssl/wolfcrypt/port/autosar/Crypto.h | 55 + .../wolfssl/wolfssl/wolfcrypt/port/autosar/Csm.h | 295 + .../wolfssl/wolfcrypt/port/autosar/StandardTypes.h | 66 + .../wolfssl/wolfcrypt/port/iotsafe/iotsafe.h | 5 + .../wolfssl/wolfssl/wolfcrypt/port/liboqs/liboqs.h | 62 + .../wolfssl/wolfssl/wolfcrypt/port/psa/psa.h | 7 - extra/wolfssl/wolfssl/wolfssl/wolfcrypt/random.h | 5 +- extra/wolfssl/wolfssl/wolfssl/wolfcrypt/rsa.h | 9 +- extra/wolfssl/wolfssl/wolfssl/wolfcrypt/settings.h | 310 +- extra/wolfssl/wolfssl/wolfssl/wolfcrypt/sha.h | 66 +- extra/wolfssl/wolfssl/wolfssl/wolfcrypt/sha256.h | 6 +- extra/wolfssl/wolfssl/wolfssl/wolfcrypt/sha512.h | 2 +- extra/wolfssl/wolfssl/wolfssl/wolfcrypt/sp_int.h | 2 +- extra/wolfssl/wolfssl/wolfssl/wolfcrypt/sphincs.h | 5 +- extra/wolfssl/wolfssl/wolfssl/wolfcrypt/tfm.h | 5 +- extra/wolfssl/wolfssl/wolfssl/wolfcrypt/types.h | 332 +- extra/wolfssl/wolfssl/wolfssl/wolfcrypt/wc_kyber.h | 280 +- extra/wolfssl/wolfssl/wolfssl/wolfcrypt/wc_lms.h | 2 +- extra/wolfssl/wolfssl/wolfssl/wolfcrypt/wc_port.h | 127 +- extra/wolfssl/wolfssl/wolfssl/wolfcrypt/xmss.h | 95 +- extra/wolfssl/wolfssl/wolfssl/wolfio.h | 12 +- extra/wolfssl/wolfssl/zephyr/CMakeLists.txt | 4 + .../zephyr/samples/wolfssl_benchmark/prj.conf | 1 + 380 files changed, 67559 insertions(+), 20107 deletions(-) create mode 100644 extra/mariabackup/aria_backup_client.cc create mode 100644 extra/mariabackup/aria_backup_client.h delete mode 100644 extra/mariabackup/changed_page_bitmap.cc delete mode 100644 extra/mariabackup/changed_page_bitmap.h create mode 100644 extra/mariabackup/common_engine.cc create mode 100644 extra/mariabackup/common_engine.h create mode 100644 extra/mariabackup/ddl_log.cc create mode 100644 extra/mariabackup/ddl_log.h create mode 100644 extra/mariabackup/encryption_plugin.cc create mode 100644 extra/mariabackup/encryption_plugin.h create mode 100644 extra/mariabackup/thread_pool.cc create mode 100644 extra/mariabackup/thread_pool.h delete mode 100644 extra/mariabackup/xb_plugin.cc delete mode 100644 extra/mariabackup/xb_plugin.h create mode 100644 extra/wolfssl/wolfssl/.github/workflows/libssh2.yml create mode 100644 extra/wolfssl/wolfssl/.github/workflows/openssh.yml create mode 100644 extra/wolfssl/wolfssl/Docker/yocto/Dockerfile create mode 100755 extra/wolfssl/wolfssl/Docker/yocto/buildAndPush.sh create mode 100644 extra/wolfssl/wolfssl/IDE/ARDUINO/Arduino_README_prepend.md create mode 100644 extra/wolfssl/wolfssl/IDE/ARDUINO/keywords.txt create mode 100644 extra/wolfssl/wolfssl/IDE/ARDUINO/library.properties.template create mode 100644 extra/wolfssl/wolfssl/IDE/ARDUINO/sketches/README.md create mode 100644 extra/wolfssl/wolfssl/IDE/ARDUINO/sketches/wolfssl_client/README.md create mode 100644 extra/wolfssl/wolfssl/IDE/ARDUINO/sketches/wolfssl_server/README.md create mode 100644 extra/wolfssl/wolfssl/IDE/ARDUINO/sketches/wolfssl_version/README.md create mode 100644 extra/wolfssl/wolfssl/IDE/ARDUINO/sketches/wolfssl_version/wolfssl_version.ino create mode 100644 extra/wolfssl/wolfssl/IDE/ARDUINO/wolfssl.h create mode 100644 extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/wolfssl_client_ESP8266.vgdbproj delete mode 100644 extra/wolfssl/wolfssl/IDE/IAR-EWARM/embOS/SAMV71_XULT/embOS_wolfcrypt_benchmark_SAMV71_XULT/settings/wolfcrypt_benchmark_Debug.jlink delete mode 100644 extra/wolfssl/wolfssl/IDE/IAR-EWARM/embOS/SAMV71_XULT/embOS_wolfcrypt_benchmark_SAMV71_XULT/wolfcrypt_benchmark.ewt delete mode 100644 extra/wolfssl/wolfssl/IDE/IAR-EWARM/embOS/SAMV71_XULT/embOS_wolfcrypt_lib_SAMV71_XULT/wolfcrypt_lib.ewt delete mode 100644 extra/wolfssl/wolfssl/IDE/IAR-EWARM/embOS/SAMV71_XULT/embOS_wolfcrypt_test_SAMV71_XULT/settings/wolfcrypt_test_Debug.jlink delete mode 100644 extra/wolfssl/wolfssl/IDE/IAR-EWARM/embOS/SAMV71_XULT/embOS_wolfcrypt_test_SAMV71_XULT/wolfcrypt_test.ewt create mode 100644 extra/wolfssl/wolfssl/IDE/MPLABX16/README.md create mode 100644 extra/wolfssl/wolfssl/IDE/MPLABX16/include.am create mode 100644 extra/wolfssl/wolfssl/IDE/MPLABX16/main.c create mode 100644 extra/wolfssl/wolfssl/IDE/MPLABX16/user_settings.h create mode 100644 extra/wolfssl/wolfssl/IDE/MPLABX16/wolfcrypt_test.X/Makefile create mode 100755 extra/wolfssl/wolfssl/IDE/MPLABX16/wolfcrypt_test.X/nbproject/configurations.xml create mode 100644 extra/wolfssl/wolfssl/IDE/MPLABX16/wolfcrypt_test.X/nbproject/include.am create mode 100644 extra/wolfssl/wolfssl/IDE/MPLABX16/wolfcrypt_test.X/nbproject/private/configurations.xml create mode 100644 extra/wolfssl/wolfssl/IDE/MPLABX16/wolfcrypt_test.X/nbproject/private/private.xml create mode 100755 extra/wolfssl/wolfssl/IDE/MPLABX16/wolfcrypt_test.X/nbproject/project.xml create mode 100644 extra/wolfssl/wolfssl/IDE/MPLABX16/wolfssl.X/Makefile create mode 100644 extra/wolfssl/wolfssl/IDE/MPLABX16/wolfssl.X/nbproject/configurations.xml create mode 100644 extra/wolfssl/wolfssl/IDE/MPLABX16/wolfssl.X/nbproject/include.am create mode 100644 extra/wolfssl/wolfssl/IDE/MPLABX16/wolfssl.X/nbproject/project.xml delete mode 100644 extra/wolfssl/wolfssl/IPP/.gitkeep create mode 100644 extra/wolfssl/wolfssl/certs/crl/crl_rsapss.pem create mode 100644 extra/wolfssl/wolfssl/certs/test-stream-sign.p7b create mode 100644 extra/wolfssl/wolfssl/examples/configs/user_settings_arduino.h create mode 100644 extra/wolfssl/wolfssl/examples/configs/user_settings_tls12.h delete mode 100644 extra/wolfssl/wolfssl/lib/dummy create mode 100644 extra/wolfssl/wolfssl/linuxkm/lkcapi_glue.c delete mode 100755 extra/wolfssl/wolfssl/pre-commit.sh delete mode 100755 extra/wolfssl/wolfssl/pre-push.sh create mode 100644 extra/wolfssl/wolfssl/wolfcrypt/src/port/autosar/README.md create mode 100644 extra/wolfssl/wolfssl/wolfcrypt/src/port/autosar/cryif.c create mode 100644 extra/wolfssl/wolfssl/wolfcrypt/src/port/autosar/crypto.c create mode 100644 extra/wolfssl/wolfssl/wolfcrypt/src/port/autosar/csm.c create mode 100644 extra/wolfssl/wolfssl/wolfcrypt/src/port/autosar/include.am create mode 100644 extra/wolfssl/wolfssl/wolfcrypt/src/port/autosar/test.c create mode 100644 extra/wolfssl/wolfssl/wolfcrypt/src/port/liboqs/liboqs.c create mode 100644 extra/wolfssl/wolfssl/wolfcrypt/src/wc_lms_impl.c create mode 100644 extra/wolfssl/wolfssl/wolfcrypt/src/wc_xmss_impl.c delete mode 100644 extra/wolfssl/wolfssl/wolfcrypt/user-crypto/Makefile.am delete mode 100644 extra/wolfssl/wolfssl/wolfcrypt/user-crypto/README.txt delete mode 100755 extra/wolfssl/wolfssl/wolfcrypt/user-crypto/autogen.sh delete mode 100644 extra/wolfssl/wolfssl/wolfcrypt/user-crypto/configure.ac delete mode 100644 extra/wolfssl/wolfssl/wolfcrypt/user-crypto/include.am delete mode 100644 extra/wolfssl/wolfssl/wolfcrypt/user-crypto/include/user_rsa.h delete mode 100644 extra/wolfssl/wolfssl/wolfcrypt/user-crypto/lib/.gitkeep delete mode 100644 extra/wolfssl/wolfssl/wolfcrypt/user-crypto/src/rsa.c create mode 100644 extra/wolfssl/wolfssl/wolfssl/wolfcrypt/port/autosar/CryIf.h create mode 100644 extra/wolfssl/wolfssl/wolfssl/wolfcrypt/port/autosar/Crypto.h create mode 100644 extra/wolfssl/wolfssl/wolfssl/wolfcrypt/port/autosar/Csm.h create mode 100644 extra/wolfssl/wolfssl/wolfssl/wolfcrypt/port/autosar/StandardTypes.h create mode 100644 extra/wolfssl/wolfssl/wolfssl/wolfcrypt/port/liboqs/liboqs.h (limited to 'extra') diff --git a/extra/CMakeLists.txt b/extra/CMakeLists.txt index 5021128e..06dc8900 100644 --- a/extra/CMakeLists.txt +++ b/extra/CMakeLists.txt @@ -13,7 +13,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include ${ZLIB_INCLUDE_DIR}) +INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include ${ZLIB_INCLUDE_DIRS}) # Default install component for the files is Server here SET(MYSQL_INSTALL_COMPONENT Server) diff --git a/extra/mariabackup/CMakeLists.txt b/extra/mariabackup/CMakeLists.txt index f1c9dca7..63ac8cf3 100644 --- a/extra/mariabackup/CMakeLists.txt +++ b/extra/mariabackup/CMakeLists.txt @@ -31,6 +31,7 @@ ENDIF() INCLUDE_DIRECTORIES( ${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/sql + ${CMAKE_SOURCE_DIR}/storage/maria ${CMAKE_CURRENT_SOURCE_DIR}/quicklz ${CMAKE_CURRENT_SOURCE_DIR} ) @@ -49,14 +50,9 @@ ADD_DEFINITIONS(-UMYSQL_SERVER) ADD_DEFINITIONS(-DPCRE_STATIC=1) ADD_DEFINITIONS(${SSL_DEFINES}) -IF(PMEM_FOUND) - ADD_COMPILE_FLAGS(xtrabackup.cc COMPILE_FLAGS "-DHAVE_PMEM") -ENDIF() - MYSQL_ADD_EXECUTABLE(mariadb-backup xtrabackup.cc innobackupex.cc - changed_page_bitmap.cc datasink.cc ds_buffer.cc ds_compress.cc @@ -72,8 +68,12 @@ MYSQL_ADD_EXECUTABLE(mariadb-backup xbstream_write.cc backup_mysql.cc backup_copy.cc - xb_plugin.cc + encryption_plugin.cc ${PROJECT_BINARY_DIR}/sql/sql_builtin.cc + aria_backup_client.cc + thread_pool.cc + ddl_log.cc + common_engine.cc ${PROJECT_SOURCE_DIR}/sql/net_serv.cc ${PROJECT_SOURCE_DIR}/libmysqld/libmysql.c COMPONENT backup @@ -82,7 +82,8 @@ MYSQL_ADD_EXECUTABLE(mariadb-backup # Export all symbols on Unix, for better crash callstacks SET_TARGET_PROPERTIES(mariadb-backup PROPERTIES ENABLE_EXPORTS TRUE) -TARGET_LINK_LIBRARIES(mariadb-backup sql sql_builtins) +TARGET_LINK_LIBRARIES(mariadb-backup sql sql_builtins aria) + IF(NOT HAVE_SYSTEM_REGEX) TARGET_LINK_LIBRARIES(mariadb-backup pcre2-posix) ENDIF() diff --git a/extra/mariabackup/aria_backup_client.cc b/extra/mariabackup/aria_backup_client.cc new file mode 100644 index 00000000..25468148 --- /dev/null +++ b/extra/mariabackup/aria_backup_client.cc @@ -0,0 +1,1016 @@ +#include +#include +extern "C" { +#include "maria_def.h" +} +#undef LSN_MAX +#include "aria_backup_client.h" +#include "backup_copy.h" +#include "common.h" +#include "sql_table.h" +#include "ma_checkpoint.h" +#include "ma_recovery.h" +#include "backup_debug.h" +#include "aria_backup.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace aria { + +const char *log_preffix = "aria_log."; + + +static std::string log_file_name_only(size_t log_num) { + std::string log_file; + { + std::stringstream ss; + ss << std::setw(8) << std::setfill('0') << log_num; + log_file.append(log_preffix).append(ss.str()); + } + return log_file; +} + + +static std::string log_file_name(const char *datadir_path, size_t log_num) { + std::string log_file(datadir_path); + return log_file.append("/").append(log_file_name_only(log_num)); +} + + +class LogFileCollection +{ + uint32 m_first; + uint32 m_count; +public: + uint32 first() const { return m_first; } + uint32 count() const { return m_count; } + uint32 last() const + { + DBUG_ASSERT(m_count > 0); + return m_first + m_count - 1; + } + + // Initialize by checking existing log files on the disk + LogFileCollection(const char *datadir, uint32 max_log_no) + { + uint32 end= find_greatest_existing_log(datadir, max_log_no); + if (!end) + { + // No log files were found at all + m_first= 0; + m_count= 0; + } + else if (end == 1) + { + // Just the very first one log file (aria_log.00000001) was found. + m_first= 1; + m_count= 1; + } + else + { + // Multiple files were found + m_first= find_greatest_missing_log(datadir, end - 1) + 1; + m_count= 1 + end - m_first; + } + } + + /* + Skip all missing log files and find the greatest existing log file, or + Skip all existing log files and find the greatest missing log file. + + @param datadir - Search files in this directory + @param start - Start searching from this log number and go downto 1. + @param kind - true - search for an existing file + false - search for a missing file. + @returns - [1..start] - the greatest found log file + of the searched kind + - 0 - if no log files of this kind + were found in the range [1..start]. + */ + static uint32 find_greatest_existing_or_missing_log(const char *datadir, + uint32 start, + bool kind) + { + DBUG_ASSERT(start > 0); + for (uint32 i= start; i > 0; i--) + { + if (file_exists(log_file_name(datadir, i).c_str()) == kind) + return i; + } + return 0; // No log files of the searched kind were found + } + + static uint32 find_greatest_existing_log(const char *datadir, uint32 start) + { + return find_greatest_existing_or_missing_log(datadir, start, true); + } + + static uint32 find_greatest_missing_log(const char *datadir, uint32 start) + { + return find_greatest_existing_or_missing_log(datadir, start, false); + } + + /* + In some scenarios (e.g. log rotate) some new log files can appear + outside of the initially assumed [first,last] log number range. + This function adds all extra files behind "last". + */ + void find_logs_after_last(const char *datadir) + { + DBUG_ASSERT(m_count > 0); + for ( ; + file_exists(log_file_name(datadir, last() + 1).c_str()) ; + m_count++) + { } + } + + void report_found(unsigned thread_num) const + { + if (m_count) + msg(thread_num, + "Found %u aria log files, " + "minimum log number %u, " + "maximum log number %u", + m_count, m_first, last()); + } + + void die_if_missing(uint32 logno) const + { + DBUG_ASSERT(logno > 0); + if (!m_count || m_first > logno || last() < logno) + die("Aria log file %u does not exists.", logno); + } +}; + + +class Table { +public: + struct Partition { + std::string m_file_path; + File m_index_file = -1; + MY_STAT m_index_file_stat; + File m_data_file = -1; + MY_STAT m_data_file_stat; + }; + Table() = default; + Table (Table &&other) = delete; + Table & operator= (Table &&other) = delete; + Table(const Table &) = delete; + Table & operator= (const Table &) = delete; + ~Table(); + bool init(const char *data_file_path); + bool open(MYSQL *con, bool opt_no_lock, unsigned thread_num); + bool close(); + bool copy(ds_ctxt_t *ds, unsigned thread_num); + + bool is_online_backup_safe() const { + DBUG_ASSERT(is_opened()); + return m_cap.online_backup_safe; + } + bool is_stats() const { + return is_stats_table(m_db.c_str(), m_table.c_str()); + } + bool is_log() const { + return is_log_table(m_db.c_str(), m_table.c_str()); + } + bool is_opened() const { + return !m_partitions.empty() && + m_partitions[0].m_index_file >= 0 && m_partitions[0].m_data_file >= 0; + }; + std::string &get_full_name() { + return m_full_name; + } + std::string &get_db() { return m_db; } + std::string &get_table() { return m_table; } + std::string &get_version() { return m_table_version; } + bool is_partitioned() const { return m_partitioned; } + void add_partition(const Table &partition) { + DBUG_ASSERT(is_partitioned()); + m_partitions.push_back(partition.m_partitions[0]); + } +#ifndef DBUG_OFF + const std::string& get_sql_name() const { return m_sql_name; } +#endif //DBUG_OFF +private: + + bool copy(ds_ctxt_t *ds, bool is_index, unsigned thread_num); + // frm and par files will be copied under BLOCK_DDL stage in + // backup_copy_non_system() + bool copy_frm_and_par(ds_ctxt_t *ds, unsigned thread_num); + bool read_table_version_id(File file); + + std::string m_db; + std::string m_table; + std::string m_full_name; + std::string m_frm_par_path; + std::string m_table_version; +#ifndef DBUG_OFF + std::string m_sql_name; +#endif //DBUG_OFF + bool m_partitioned = false; + std::vector m_partitions; + ARIA_TABLE_CAPABILITIES m_cap; +}; + +Table::~Table() { + (void)close(); +} + +bool Table::init(const char *data_file_path) { + DBUG_ASSERT(data_file_path); + + const char *ext_pos = strrchr(data_file_path, '.'); + if (!ext_pos) + return false; + + char db_name_orig[FN_REFLEN]; + char table_name_orig[FN_REFLEN]; + parse_db_table_from_file_path( + data_file_path, db_name_orig, table_name_orig); + if (!db_name_orig[0] || !table_name_orig[0]) + return false; + char db_name_conv[FN_REFLEN]; + char table_name_conv[FN_REFLEN]; + filename_to_tablename(db_name_orig, db_name_conv, sizeof(db_name_conv)); + filename_to_tablename( + table_name_orig, table_name_conv, sizeof(table_name_conv)); + if (!db_name_conv[0] || !table_name_conv[0]) + return false; + + if (strstr(data_file_path, "#P#")) + m_partitioned = true; + + const char *table_name_begin = strrchr(data_file_path, FN_LIBCHAR); + if (!table_name_begin) + return false; + m_frm_par_path.assign(data_file_path, table_name_begin + 1). + append(table_name_orig); + + m_db.assign(db_name_conv); + m_table.assign(table_name_conv); + // TODO: find the correct way to represent quoted table/db names + m_full_name.assign("`").append(m_db).append("`.`"). + append(m_table).append("`"); +#ifndef DBUG_OFF + m_sql_name.assign(m_db).append("/").append(m_table); +#endif // DBUG_OFF + Partition partition; + partition.m_file_path.assign(data_file_path, ext_pos - data_file_path); + m_partitions.push_back(std::move(partition)); + return true; +} + +bool Table::read_table_version_id(File file) { + m_table_version = ::read_table_version_id(file); + return m_table_version.empty(); +} + +bool Table::open(MYSQL *con, bool opt_no_lock, unsigned thread_num) { + int error= 1; + bool have_capabilities = false; + File frm_file = -1; + + if (!opt_no_lock && !backup_lock(con, m_full_name.c_str())) { + msg(thread_num, "Error on BACKUP LOCK for aria table %s", + m_full_name.c_str()); + goto exit; + } + + for (Partition &partition : m_partitions) { + std::string file_path = partition.m_file_path + ".MAI"; + if ((partition.m_index_file= my_open(file_path.c_str(), + O_RDONLY | O_SHARE | O_NOFOLLOW | O_CLOEXEC, + MYF(MY_WME))) < 0) { + msg(thread_num, "Error on aria table file open %s", file_path.c_str()); + goto exit; + } + if (!my_stat(file_path.c_str(), &partition.m_index_file_stat, MYF(0))) { + msg(thread_num, "Error on aria table file stat %s", file_path.c_str()); + goto exit; + } + if (!have_capabilities) { + if ((error= aria_get_capabilities(partition.m_index_file, &m_cap))) { + msg(thread_num, "aria_get_capabilities failed: %d", error); + goto exit; + } + have_capabilities = true; + } + + file_path = partition.m_file_path + ".MAD"; + if ((partition.m_data_file= my_open(file_path.c_str(), + O_RDONLY | O_SHARE | O_NOFOLLOW | O_CLOEXEC, MYF(MY_WME))) < 0) { + msg(thread_num, "Error on aria table file open %s", file_path.c_str()); + goto exit; + } + if (!my_stat(file_path.c_str(), &partition.m_data_file_stat, MYF(0))) { + msg(thread_num, "Error on aria table file stat %s", file_path.c_str()); + goto exit; + } + } + + if ((frm_file = mysql_file_open( + key_file_frm, (m_frm_par_path + ".frm").c_str(), + O_RDONLY | O_SHARE, MYF(0))) < 0) { + msg(thread_num, "Error on aria table %s file open", + (m_frm_par_path + ".frm").c_str()); + goto exit; + } + + error = 0; + +exit: + if (!opt_no_lock && !backup_unlock(con)) { + msg(thread_num, "Error on BACKUP UNLOCK for aria table %s", + m_full_name.c_str()); + error = 1; + } + if (error) + (void)close(); + else { + (void)read_table_version_id(frm_file); + mysql_file_close(frm_file, MYF(MY_WME)); + } + return !error; +} + +bool Table::close() { + for (Partition &partition : m_partitions) { + if (partition.m_index_file >= 0) { + my_close(partition.m_index_file, MYF(MY_WME)); + partition.m_index_file = -1; + } + if (partition.m_data_file >= 0) { + my_close(partition.m_data_file, MYF(MY_WME)); + partition.m_data_file = -1; + } + } + return true; +} + +bool Table::copy(ds_ctxt_t *ds, unsigned thread_num) { + DBUG_ASSERT(is_opened()); + DBUG_MARIABACKUP_EVENT_LOCK("before_aria_table_copy", + fil_space_t::name_type(m_sql_name.data(), m_sql_name.size())); + bool result = +// copy_frm_and_par(ds, thread_num) && + copy(ds, true, thread_num) && copy(ds, false, thread_num); + return result; +} + +bool Table::copy(ds_ctxt_t *ds, bool is_index, unsigned thread_num) { + DBUG_ASSERT(ds); + const char *ext = is_index ? ".MAI" : ".MAD"; + int error= 1; + for (const Partition &partition : m_partitions) { + ds_file_t *dst_file = nullptr; + uchar *copy_buffer = nullptr; + std::string full_name = partition.m_file_path + ext; + const char *dst_path = + (xtrabackup_copy_back || xtrabackup_move_back) ? + full_name.c_str() : trim_dotslash(full_name.c_str()); + + dst_file = ds_open(ds, dst_path, + is_index ? &partition.m_index_file_stat : &partition.m_data_file_stat); + if (!dst_file) { + msg(thread_num, "error: cannot open the destination stream for %s", + dst_path); + goto err; + } + + copy_buffer = + reinterpret_cast(my_malloc(PSI_NOT_INSTRUMENTED, + m_cap.block_size, MYF(0))); + + DBUG_MARIABACKUP_EVENT_LOCK( + is_index ? + "before_aria_index_file_copy": + "before_aria_data_file_copy", + fil_space_t::name_type(m_sql_name.data(), + m_sql_name.size())); + + for (ulonglong block= 0 ; ; block++) { + size_t length = m_cap.block_size; + if (is_index) { + if ((error= aria_read_index( + partition.m_index_file, &m_cap, block, copy_buffer) == + HA_ERR_END_OF_FILE)) + break; + } else { + if ((error= aria_read_data( + partition.m_data_file, &m_cap, block, copy_buffer, &length) == + HA_ERR_END_OF_FILE)) + break; + } + if (error) { + msg(thread_num, "error: aria_read %s failed: %d", + is_index ? "index" : "data", error); + goto err; + } + xtrabackup_io_throttling(); + if ((error = ds_write(dst_file, copy_buffer, length))) { + msg(thread_num, "error: aria_write failed: %d", error); + goto err; + } + } + + DBUG_MARIABACKUP_EVENT_LOCK( + is_index ? + "after_aria_index_file_copy": + "after_aria_data_file_copy", + fil_space_t::name_type(m_sql_name.data(), + m_sql_name.size())); + + error = 0; + msg(thread_num, "aria table file %s is copied successfully.", + full_name.c_str()); + + err: + if (dst_file) + ds_close(dst_file); + if (copy_buffer) + my_free(copy_buffer); + if (error) + break; + } + return !error; +} + +class BackupImpl { +public: + BackupImpl( + const char *datadir_path, + const char *aria_log_path, + ds_ctxt_t *datasink, bool opt_no_lock, + std::vector &con_pool, ThreadPool &thread_pool) : + m_datadir_path(datadir_path), + m_aria_log_dir_path(aria_log_path), + m_ds(datasink), m_con_pool(con_pool), + m_tasks_group(thread_pool), m_thread_pool(thread_pool) { } + ~BackupImpl() { destroy(); } + bool init(); + bool start(bool no_lock); + bool wait_for_finish(); + bool copy_offline_tables( + const std::unordered_set *exclude_tables, bool no_lock, + bool copy_stats); + bool finalize(); + void set_post_copy_table_hook(const post_copy_table_hook_t &hook) { + m_table_post_copy_hook = hook; + } + bool copy_log_tail() { return copy_log_tail(0, false); } +private: + void destroy(); + void scan_job(bool no_lock, unsigned thread_num); + bool copy_log_tail(unsigned thread_num, bool finalize); + void copy_log_file_job(size_t log_num, unsigned thread_num); + void destroy_log_tail(); + void process_table_job(Table *table, bool online_only, bool copy_stats, + bool no_lock, unsigned thread_num); + + const char *m_datadir_path; + const char *m_aria_log_dir_path; + std::string aria_log_dir_path() const + { + if (!m_aria_log_dir_path || !m_aria_log_dir_path[0]) + return m_datadir_path; + if (is_absolute_path(m_aria_log_dir_path)) + return m_aria_log_dir_path; + return std::string(m_datadir_path).append("/") + .append(m_aria_log_dir_path); + } + ds_ctxt_t *m_ds; + std::vector &m_con_pool; + + TasksGroup m_tasks_group; + + std::mutex m_offline_tables_mutex; + std::vector> m_offline_tables; + post_copy_table_hook_t m_table_post_copy_hook; + + ThreadPool &m_thread_pool; + + size_t m_last_log_num = 0; + ds_file_t* m_last_log_dst = nullptr; + File m_last_log_src = -1; +}; + +bool BackupImpl::init() { + DBUG_ASSERT(m_tasks_group.is_finished()); + return true; +}; + +void BackupImpl::destroy() { + DBUG_ASSERT(m_tasks_group.is_finished()); + destroy_log_tail(); +} + +bool BackupImpl::start(bool no_lock) { + DBUG_ASSERT(m_tasks_group.is_finished()); + m_tasks_group.push_task( + std::bind(&BackupImpl::scan_job, this, no_lock, std::placeholders::_1)); + return true; +} + +void BackupImpl::process_table_job( + Table *table_ptr, bool online_only, bool copy_stats, bool no_lock, + unsigned thread_num) { + DBUG_ASSERT(table_ptr); + DBUG_ASSERT(thread_num < m_con_pool.size()); + std::unique_ptr table(table_ptr); + bool is_online; + bool is_stats; + bool need_copy; + int result = 1; + + if (!m_tasks_group.get_result()) + goto exit; + + if (!table->open(m_con_pool[thread_num], no_lock, thread_num)) { + // if table can't be opened, it might be removed or renamed, this is not + // error for transactional tables + table->close(); // Close opened table files + goto exit; + } + + is_online = table->is_online_backup_safe(); + is_stats = table->is_stats(); + + need_copy = (!online_only || is_online) && (copy_stats || !is_stats); + + if (need_copy && !table->copy(m_ds, thread_num)) { + table->close(); + DBUG_MARIABACKUP_EVENT_LOCK("after_aria_table_copy", + fil_space_t::name_type(table->get_sql_name().data(), + table->get_sql_name().size())); + // if table is opened, it must be copied, + // the corresponding diagnostic messages must be issued in Table::copy() + result = 0; + goto exit; + } + + if (!table->close()) { + msg(thread_num, "Can't close aria table %s.\n", + table->get_full_name().c_str()); + result = 0; + goto exit; + } + + if (!need_copy) { + std::lock_guard lock(m_offline_tables_mutex); + m_offline_tables.push_back(std::move(table)); + } + else { + DBUG_MARIABACKUP_EVENT_LOCK("after_aria_table_copy", + fil_space_t::name_type(table->get_sql_name().data(), + table->get_sql_name().size())); + if (m_table_post_copy_hook) + m_table_post_copy_hook( + std::move(table->get_db()), + std::move(table->get_table()), + std::move(table->get_version())); + } +exit: + m_tasks_group.finish_task(result); +} + + +void BackupImpl::scan_job(bool no_lock, unsigned thread_num) { + std::unordered_map> partitioned_tables; + + std::string aria_log_dir_path_cache(aria_log_dir_path()); + std::string log_control_file_path(aria_log_dir_path_cache); + log_control_file_path.append("/aria_log_control"); + if (!m_ds->copy_file( + log_control_file_path.c_str(), "aria_log_control", + 0, false)) { + msg("Aria log control file copying error."); + m_tasks_group.finish_task(0); + return; + } + + msg(thread_num, "Loading aria_log_control."); + aria_readonly= 1; + maria_data_root= aria_log_dir_path_cache.c_str(); + if (ma_control_file_open(FALSE, FALSE, FALSE, O_RDONLY)) + die("Can't open Aria control file (%d)", errno); + uint32 aria_log_control_last_log_number= last_logno; + msg(thread_num, "aria_log_control: last_log_number: %d", + aria_log_control_last_log_number); + ma_control_file_end(); + + msg(thread_num, "Start scanning aria tables."); + + foreach_file_in_db_dirs(m_datadir_path, [&](const char *file_path)->bool { + + if (check_if_skip_table(file_path)) { + msg(thread_num, "Skipping %s.", file_path); + return true; + } + + if (!ends_with(file_path, ".MAD")) + return true; + + std::unique_ptr
table(new Table()); + if (!table->init(file_path)) { + msg(thread_num, "Can't init aria table %s.\n", file_path); + return true; + } + + if (table->is_log()) + return true; + + if (table->is_partitioned()) { + auto table_it = partitioned_tables.find(table->get_full_name()); + if (table_it == partitioned_tables.end()) { + partitioned_tables[table->get_full_name()] = std::move(table); + } else { + table_it->second->add_partition(*table); + } + return true; + } + + m_tasks_group.push_task( + std::bind(&BackupImpl::process_table_job, this, table.release(), true, + false, no_lock, std::placeholders::_1)); + return true; + }); + + for (auto &table_it : partitioned_tables) { + m_tasks_group.push_task( + std::bind(&BackupImpl::process_table_job, this, table_it.second.release(), + true, false, no_lock, std::placeholders::_1)); + } + + msg(thread_num, "Start scanning aria log files."); + + LogFileCollection logs(aria_log_dir_path_cache.c_str(), + aria_log_control_last_log_number); + logs.report_found(thread_num); + logs.die_if_missing(aria_log_control_last_log_number); + + m_last_log_num= logs.last(); + + DBUG_MARIABACKUP_EVENT("after_scanning_log_files", {}); + + for (uint32 i= logs.first(); i <= logs.last(); ++i) + m_tasks_group.push_task( + std::bind(&BackupImpl::copy_log_file_job, this, + i, std::placeholders::_1)); + + msg(thread_num, "Stop scanning aria tables."); + + m_tasks_group.finish_task(1); +} + +template +T align_down(T n, ulint align_no) +{ + DBUG_ASSERT(align_no > 0); + DBUG_ASSERT(ut_is_2pow(align_no)); + return n & ~(static_cast(align_no) - 1); +} + +static ssize_t copy_file_chunk(File src, ds_file_t* dst, size_t size) { + size_t bytes_read; + static const size_t max_buf_size = 10 * 1024 * 1024; + size_t buf_size = size ? std::min(size, max_buf_size) : max_buf_size; + std::unique_ptr buf(new uchar[buf_size]); + ssize_t copied_size = 0; + bool unlim = !size; + while((unlim || size) && (bytes_read = my_read(src, buf.get(), + unlim ? buf_size : std::min(buf_size, size), MY_WME))) { + if (bytes_read == size_t(-1)) + return -1; + xtrabackup_io_throttling(); + if (ds_write(dst, buf.get(), bytes_read)) + return -1; + copied_size += bytes_read; + if (!unlim) + size -= bytes_read; + } + return copied_size; +} + +bool BackupImpl::copy_log_tail(unsigned thread_num, bool finalize) { + bool result = false; + std::string log_file = log_file_name(aria_log_dir_path().c_str(), m_last_log_num); + std::string prev_log_file; + ssize_t total_bytes_copied = 0; + MY_STAT stat_info; + my_off_t file_offset = 0; + size_t to_copy_size = 0; + +repeat: + memset(&stat_info, 0, sizeof(MY_STAT)); + if (!m_tasks_group.get_result()) { + msg(thread_num, "Skip copying aria lof file tail %s due to error.", + log_file.c_str()); + result = true; + goto exit; + } + + msg(thread_num, "Start copying aria log file tail: %s", log_file.c_str()); + + if (m_last_log_src < 0 && (m_last_log_src = + my_open(log_file.c_str(), O_RDONLY | O_SHARE | O_NOFOLLOW | O_CLOEXEC, + MYF(MY_WME))) < 0) { + msg("Aria log file %s open failed: %d", log_file.c_str(), my_errno); + goto exit; + } + + if (!m_last_log_dst && + !(m_last_log_dst = ds_open(m_ds, + log_file_name_only(m_last_log_num).c_str(), + &stat_info, false))) { + msg(thread_num, "error: failed to open the target stream for " + "aria log file %s.", + log_file.c_str()); + goto exit; + } + +// If there is no need to finalize log file copying, calculate the size to copy +// without the last page, which can be rewritten by the server +// (see translog_force_current_buffer_to_finish()). + if (!finalize) { + if (my_fstat(m_last_log_src, &stat_info, MYF(0))) { + msg(thread_num, "error: failed to get file size for aria log file: %s.", + log_file.c_str()); + goto exit; + } + if ((file_offset = my_tell(m_last_log_src, MYF(0))) == (my_off_t)(-1)) { + msg(thread_num, "error: failed to get file offset for aria log file: %s.", + log_file.c_str()); + goto exit; + } + DBUG_ASSERT(file_offset <= static_cast(stat_info.st_size)); + to_copy_size = static_cast(stat_info.st_size - file_offset); + to_copy_size = to_copy_size >= TRANSLOG_PAGE_SIZE ? + (align_down(to_copy_size, TRANSLOG_PAGE_SIZE) - TRANSLOG_PAGE_SIZE) : 0; + } + +// Copy from the last position to the end of file, +// excluding the last page is there is no need to finalize the copy. + if ((to_copy_size || finalize) && + (total_bytes_copied = copy_file_chunk(m_last_log_src, + m_last_log_dst, to_copy_size)) < 0) { + msg(thread_num, "Aria log file %s chunk copy error", log_file.c_str()); + goto exit; + } + + msg(thread_num, "Stop copying aria log file tail: %s, copied %zu bytes", + log_file.c_str(), total_bytes_copied); + +// Check if there is new log file, if yes, then copy the last page of the old +// one, and fix it last LSN in the log header, as it is changed on new +// log file creating by the server (see translog_create_new_file() and +// translog_max_lsn_to_header()). Then close the old log file and repeat +// the copying for the new log file. + prev_log_file = std::move(log_file); + log_file = log_file_name(aria_log_dir_path().c_str(), m_last_log_num + 1); + if (file_exists(log_file.c_str())) { + uchar lsn_buff[LSN_STORE_SIZE]; + msg(thread_num, "Found new aria log tail file: %s, start copy %s tail", + log_file.c_str(), prev_log_file.c_str()); + if ((total_bytes_copied = copy_file_chunk(m_last_log_src, + m_last_log_dst, 0)) < 0) { + msg(thread_num, "Aria log file %s tail copy error", + prev_log_file.c_str()); + goto exit; + } + + if (my_pread(m_last_log_src, lsn_buff, LSN_STORE_SIZE, + (LOG_HEADER_DATA_SIZE - LSN_STORE_SIZE), MYF(0)) < LSN_STORE_SIZE) { + msg(thread_num, "Aria lsn store read error for log file %s", + prev_log_file.c_str()); + goto exit; + } + + if (ds_seek_set(m_last_log_dst, (LOG_HEADER_DATA_SIZE - LSN_STORE_SIZE))) { + msg(thread_num, "Set aria log pointer error for log file %s", + prev_log_file.c_str()); + goto exit; + } + + if (ds_write(m_last_log_dst, lsn_buff, LSN_STORE_SIZE)) { + msg(thread_num, "LSN write error for aria log file %s", + prev_log_file.c_str()); + goto exit; + } + + msg(thread_num, "The last %zu bytes were copied for %s.", + total_bytes_copied, prev_log_file.c_str()); + destroy_log_tail(); + ++m_last_log_num; + goto repeat; + } + + result = true; + +exit: + if (!result) + destroy_log_tail(); + return result; +} + +void BackupImpl::copy_log_file_job(size_t log_num, unsigned thread_num) { + DBUG_ASSERT(log_num <= m_last_log_num); + + if (!m_tasks_group.get_result()) { + msg(thread_num, "Skip copying %zu aria log file due to error", log_num); + m_tasks_group.finish_task(0); + return; + } + +// Copy log file if the file is not the last one. + if (log_num < m_last_log_num) { + std::string log_file = log_file_name(aria_log_dir_path().c_str(), log_num); + if (!m_ds->copy_file(log_file.c_str(), + log_file_name_only(log_num).c_str(), + thread_num, false)) { + msg(thread_num, "Error on copying %s aria log file.", log_file.c_str()); + m_tasks_group.finish_task(0); + } + else + m_tasks_group.finish_task(1); + return; + } +// Copy the last log file. + m_tasks_group.finish_task(copy_log_tail(thread_num, false) ? 1 : 0); +} + +void BackupImpl::destroy_log_tail() { + if (m_last_log_src >= 0) { + my_close(m_last_log_src, MYF(MY_WME)); + m_last_log_src = -1; + } + if (m_last_log_dst) { + ds_close(m_last_log_dst); + m_last_log_dst = nullptr; + } +} + +bool BackupImpl::wait_for_finish() { + return m_tasks_group.wait_for_finish(); +} + +bool BackupImpl::copy_offline_tables( + const std::unordered_set *exclude_tables, bool no_lock, + bool copy_stats) { + DBUG_ASSERT(m_tasks_group.is_finished()); + + std::vector> ignored_tables; + + while (true) { + std::unique_lock lock(m_offline_tables_mutex); + if (m_offline_tables.empty()) + break; + auto table = std::move(m_offline_tables.back()); + m_offline_tables.pop_back(); + lock.unlock(); + if ((exclude_tables && + exclude_tables->count(table_key(table->get_db(), table->get_table()))) || + (!copy_stats && table->is_stats())) { + ignored_tables.push_back(std::move(table)); + continue; + } + m_tasks_group.push_task( + std::bind(&BackupImpl::process_table_job, this, table.release(), false, + copy_stats, no_lock, std::placeholders::_1)); + } + + if (!ignored_tables.empty()) { + std::lock_guard lock(m_offline_tables_mutex); + m_offline_tables = std::move(ignored_tables); + } + + return true; +} + +bool BackupImpl::finalize() { + DBUG_ASSERT(m_tasks_group.is_finished()); + DBUG_ASSERT(!m_con_pool.empty()); + bool result = true; + msg("Start copying statistics aria tables."); + copy_offline_tables(nullptr, true, true); + while (!m_tasks_group.is_finished()) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + msg("Stop copying statistics aria tables."); + copy_log_tail(0, true); + destroy_log_tail(); + return result; +} + +Backup::Backup(const char *datadir_path, + const char *aria_log_path, + ds_ctxt_t *datasink, + std::vector &con_pool, ThreadPool &thread_pool) : + m_backup_impl( + new BackupImpl(datadir_path, aria_log_path, + datasink, opt_no_lock, con_pool, + thread_pool)) { } + +Backup::~Backup() { + delete m_backup_impl; +} + +bool Backup::init() { + return m_backup_impl->init(); +} + +bool Backup::start(bool no_lock) { + return m_backup_impl->start(no_lock); +} + +bool Backup::wait_for_finish() { + return m_backup_impl->wait_for_finish(); +} + +bool Backup::copy_offline_tables( + const std::unordered_set *exclude_tables, bool no_lock, + bool copy_stats) { + return m_backup_impl->copy_offline_tables(exclude_tables, no_lock, + copy_stats); +} + +bool Backup::finalize() { + return m_backup_impl->finalize(); +} + +bool Backup::copy_log_tail() { + return m_backup_impl->copy_log_tail(); +} + +void Backup::set_post_copy_table_hook(const post_copy_table_hook_t &hook) { + m_backup_impl->set_post_copy_table_hook(hook); +} + +bool prepare(const char *target_dir) { + maria_data_root= (char *)target_dir; + + if (maria_init()) + die("Can't init Aria engine (%d)", errno); + + maria_block_size= 0; /* Use block size from file */ + /* we don't want to create a control file, it MUST exist */ + if (ma_control_file_open(FALSE, TRUE, TRUE, control_file_open_flags)) + die("Can't open Aria control file (%d)", errno); + + if (last_logno == FILENO_IMPOSSIBLE) + die("Can't find any Aria log"); + + LogFileCollection logs(target_dir, last_logno); + logs.die_if_missing(last_logno); // Fatal, a broken backup. + /* + "mariadb-backup --backup" can put extra log files, + with log number greater than last_logno. For example, + this combination of files is possible: + - aria_log_control (with last_logno==1) + - aria_log.00000001 (last_logno) + - aria_log.00000002 (last_logno+1, the extra log file) + This can happen if during the ealier run of + "mariadb-backup --backup" a log rotate happened. + The extra log file is copied to the backup directory, + but last_logno in aria_log_control does not get updated. + This mismatch is probably not good and should eventually be fixed. + But during "mariadb-backup --prepare" this mismatch goes away: + aria_log_control gets fixed to say last_logno==2. + See mysql-test/suite/mariabackup/aria_log_rotate_during_backup.test, + it covers the scenario with one extra file created during --backup. + */ + logs.find_logs_after_last(target_dir); + last_logno= logs.last(); // Update last_logno if extra logs were found + + if (init_pagecache(maria_pagecache, 1024L*1024L, 0, 0, + static_cast(maria_block_size), 0, MY_WME) == 0) + die("Got error in Aria init_pagecache() (errno: %d)", errno); + + if (init_pagecache(maria_log_pagecache, 1024L*1024L, + 0, 0, TRANSLOG_PAGE_SIZE, 0, MY_WME) == 0 || + translog_init(maria_data_root, TRANSLOG_FILE_SIZE, + 0, 0, maria_log_pagecache, TRANSLOG_DEFAULT_FLAGS, FALSE)) + die("Can't init Aria loghandler (%d)", errno); + + if (maria_recovery_from_log()) + die("Aria log apply FAILED"); + + if (maria_recovery_changed_data || recovery_failures) { + if (ma_control_file_write_and_force(last_checkpoint_lsn, last_logno, + max_trid_in_control_file, 0)) + die("Aria control file update error"); +// TODO: find out do we need checkpoint here + } + + maria_end(); + return true; +} + +} // namespace aria diff --git a/extra/mariabackup/aria_backup_client.h b/extra/mariabackup/aria_backup_client.h new file mode 100644 index 00000000..7a581b58 --- /dev/null +++ b/extra/mariabackup/aria_backup_client.h @@ -0,0 +1,38 @@ +#pragma once +#include "my_global.h" +#include "datasink.h" +#include "backup_mysql.h" +#include "thread_pool.h" +#include "xtrabackup.h" + +namespace aria { + +bool prepare(const char *target_dir); + +class BackupImpl; + +class Backup { + public: + Backup(const char *datadir_path, + const char *aria_log_path, + ds_ctxt_t *datasink, + std::vector &con_pool, ThreadPool &thread_pool); + ~Backup(); + Backup (Backup &&other) = delete; + Backup & operator= (Backup &&other) = delete; + Backup(const Backup &) = delete; + Backup & operator= (const Backup &) = delete; + bool init(); + bool start(bool no_lock); + bool wait_for_finish(); + bool copy_offline_tables( + const std::unordered_set *exclude_tables, bool no_lock, + bool copy_stats); + bool finalize(); + bool copy_log_tail(); + void set_post_copy_table_hook(const post_copy_table_hook_t &hook); + private: + BackupImpl *m_backup_impl; +}; + +} // namespace aria diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index f8d315d9..198da01a 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -41,6 +41,9 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA *******************************************************/ #include +#include +#include +#include #include #include #include @@ -66,19 +69,26 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA #include #endif +#ifdef MYSQL_CLIENT +#define WAS_MYSQL_CLIENT 1 +#undef MYSQL_CLIENT +#endif + +#include "table.h" + +#ifdef WAS_MYSQL_CLIENT +#define MYSQL_CLIENT 1 +#undef WAS_MYSQL_CLIENT +#endif #define ROCKSDB_BACKUP_DIR "#rocksdb" -/* list of files to sync for --rsync mode */ -static std::set rsync_list; /* locations of tablespaces read from .isl files */ static std::map tablespace_locations; /* Whether LOCK BINLOG FOR BACKUP has been issued during backup */ bool binlog_locked; -static void rocksdb_create_checkpoint(); -static bool has_rocksdb_plugin(); static void rocksdb_backup_checkpoint(ds_ctxt *ds_data); static void rocksdb_copy_back(ds_ctxt *ds_data); @@ -135,10 +145,6 @@ struct datadir_thread_ctxt_t { bool ret; }; -static bool backup_files_from_datadir(ds_ctxt_t *ds_data, - const char *dir_path, - const char *prefix); - /************************************************************************ Retirn true if character if file separator */ bool @@ -585,7 +591,6 @@ datafile_read(datafile_cur_t *cursor) Check to see if a file exists. Takes name of the file to check. @return true if file exists. */ -static bool file_exists(const char *filename) { @@ -601,7 +606,6 @@ file_exists(const char *filename) /************************************************************************ Trim leading slashes from absolute path so it becomes relative */ -static const char * trim_dotslash(const char *path) { @@ -634,7 +638,7 @@ ends_with(const char *str, const char *suffix) && strcmp(str + str_len - suffix_len, suffix) == 0); } -static bool starts_with(const char *str, const char *prefix) +bool starts_with(const char *str, const char *prefix) { return strncmp(str, prefix, strlen(prefix)) == 0; } @@ -785,7 +789,6 @@ directory_exists_and_empty(const char *dir, const char *comment) /************************************************************************ Check if file name ends with given set of suffixes. @return true if it does. */ -static bool filename_matches(const char *filename, const char **ext_list) { @@ -800,52 +803,127 @@ filename_matches(const char *filename, const char **ext_list) return(false); } - -/************************************************************************ -Copy data file for backup. Also check if it is allowed to copy by -comparing its name to the list of known data file types and checking -if passes the rules for partial backup. -@return true if file backed up or skipped successfully. */ +// TODO: the code can be used to find storage engine of partitions +/* static -bool -datafile_copy_backup(ds_ctxt *ds_data, const char *filepath, uint thread_n) -{ - const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI", - "MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par", - NULL}; +bool is_aria_frm_or_par(const char *path) { + if (!ends_with(path, ".frm") && !ends_with(path, ".par")) + return false; - /* Get the name and the path for the tablespace. node->name always - contains the path (which may be absolute for remote tablespaces in - 5.6+). space->name contains the tablespace name in the form - "./database/table.ibd" (in 5.5-) or "database/table" (in 5.6+). For a - multi-node shared tablespace, space->name contains the name of the first - node, but that's irrelevant, since we only need node_name to match them - against filters, and the shared tablespace is always copied regardless - of the filters value. */ + const char *frm_path = path; + if (ends_with(path, ".par")) { + size_t frm_path_len = strlen(path); + DBUG_ASSERT(frm_path_len > strlen("frm")); + frm_path = strdup(path); + strcpy(const_cast(frm_path) + frm_path_len - strlen("frm"), "frm"); + } - if (check_if_skip_table(filepath)) { - msg(thread_n,"Skipping %s.", filepath); - return(true); + bool result = false; + File file; + uchar header[40]; + legacy_db_type dbt; + + if ((file= mysql_file_open(key_file_frm, frm_path, O_RDONLY | O_SHARE, MYF(0))) + < 0) + goto err; + + if (mysql_file_read(file, (uchar*) header, sizeof(header), MYF(MY_NABP))) + goto err; + + if (!strncmp((char*) header, "TYPE=VIEW\n", 10)) + goto err; + + if (!is_binary_frm_header(header)) + goto err; + + dbt = (legacy_db_type)header[3]; + + if (dbt == DB_TYPE_ARIA) { + result = true; } + else if (dbt == DB_TYPE_PARTITION_DB) { + MY_STAT state; + uchar *frm_image= 0; +// uint n_length; - if (filename_matches(filepath, ext_list)) { - return ds_data->copy_file(filepath, filepath, thread_n); + if (mysql_file_fstat(file, &state, MYF(MY_WME))) + goto err; + + if (mysql_file_seek(file, 0, SEEK_SET, MYF(MY_WME))) + goto err; + + if (read_string(file, &frm_image, (size_t)state.st_size)) + goto err; + + dbt = (legacy_db_type)frm_image[61]; + if (dbt == DB_TYPE_ARIA) { + result = true; + } + my_free(frm_image); } - return(true); +err: + if (file >= 0) + mysql_file_close(file, MYF(MY_WME)); + if (frm_path != path) + free(const_cast(frm_path)); + return result; } +*/ +void parse_db_table_from_file_path( + const char *filepath, char *dbname, char *tablename) { + dbname[0] = '\0'; + tablename[0] = '\0'; + const char *dbname_start = nullptr; + const char *tablename_start = filepath; + const char *const_ptr; + while ((const_ptr = strchr(tablename_start, FN_LIBCHAR)) != NULL) { + dbname_start = tablename_start; + tablename_start = const_ptr + 1; + } + if (!dbname_start) + return; + size_t dbname_len = tablename_start - dbname_start - 1; + if (dbname_len >= FN_REFLEN) + dbname_len = FN_REFLEN-1; + strmake(dbname, dbname_start, dbname_len); + strmake(tablename, tablename_start, FN_REFLEN-1); + char *ptr; + if ((ptr = strchr(tablename, '.'))) + *ptr = '\0'; + if ((ptr = strstr(tablename, "#P#"))) + *ptr = '\0'; +} + +bool is_system_table(const char *dbname, const char *tablename) +{ + DBUG_ASSERT(dbname); + DBUG_ASSERT(tablename); + + LEX_CSTRING lex_dbname; + LEX_CSTRING lex_tablename; + lex_dbname.str = dbname; + lex_dbname.length = strlen(dbname); + lex_tablename.str = tablename; + lex_tablename.length = strlen(tablename); + + TABLE_CATEGORY tg = get_table_category(&lex_dbname, &lex_tablename); + + return (tg == TABLE_CATEGORY_LOG) || (tg == TABLE_CATEGORY_SYSTEM); +} /************************************************************************ -Same as datafile_copy_backup, but put file name into the list for -rsync command. */ +Copy data file for backup. Also check if it is allowed to copy by +comparing its name to the list of known data file types and checking +if passes the rules for partial backup. +@return true if file backed up or skipped successfully. */ static bool -datafile_rsync_backup(const char *filepath, bool save_to_list, FILE *f) +datafile_copy_backup(ds_ctxt *ds_data, const char *filepath, uint thread_n) { - const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI", - "MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par", - NULL}; + const char *ext_list[] = {".frm", ".isl", ".TRG", ".TRN", ".opt", ".par", + NULL}; /* Get the name and the path for the tablespace. node->name always contains the path (which may be absolute for remote tablespaces in @@ -857,15 +935,13 @@ datafile_rsync_backup(const char *filepath, bool save_to_list, FILE *f) of the filters value. */ if (check_if_skip_table(filepath)) { + msg(thread_n,"Skipping %s.", filepath); return(true); } if (filename_matches(filepath, ext_list)) { - fprintf(f, "%s\n", filepath); - if (save_to_list) { - rsync_list.insert(filepath); - } - } + return ds_data->copy_file(filepath, filepath, thread_n); + } return(true); } @@ -1004,16 +1080,15 @@ Copy file for backup/restore. bool ds_ctxt_t::copy_file(const char *src_file_path, const char *dst_file_path, - uint thread_n) + uint thread_n, + bool rewrite) { char dst_name[FN_REFLEN]; ds_file_t *dstfile = NULL; datafile_cur_t cursor; xb_fil_cur_result_t res; DBUG_ASSERT(datasink->remove); - const char *dst_path = - (xtrabackup_copy_back || xtrabackup_move_back)? - dst_file_path : trim_dotslash(dst_file_path); + const char *dst_path = convert_dst(dst_file_path); if (!datafile_open(src_file_path, &cursor, thread_n)) { goto error_close; @@ -1021,7 +1096,7 @@ ds_ctxt_t::copy_file(const char *src_file_path, strncpy(dst_name, cursor.rel_path, sizeof(dst_name)); - dstfile = ds_open(this, dst_path, &cursor.statinfo); + dstfile = ds_open(this, dst_path, &cursor.statinfo, rewrite); if (dstfile == NULL) { msg(thread_n,"error: " "cannot open the destination stream for %s", dst_name); @@ -1245,278 +1320,45 @@ cleanup: } - - -static bool -backup_files(ds_ctxt *ds_data, const char *from, bool prep_mode) +backup_files(ds_ctxt *ds_data, const char *from) { - char rsync_tmpfile_name[FN_REFLEN]; - FILE *rsync_tmpfile = NULL; datadir_iter_t *it; datadir_node_t node; bool ret = true; - - if (prep_mode && !opt_rsync) { - return(true); - } - - if (opt_rsync) { - snprintf(rsync_tmpfile_name, sizeof(rsync_tmpfile_name), - "%s/%s%d", opt_mysql_tmpdir, - "xtrabackup_rsyncfiles_pass", - prep_mode ? 1 : 2); - rsync_tmpfile = fopen(rsync_tmpfile_name, "w"); - if (rsync_tmpfile == NULL) { - msg("Error: can't create file %s", - rsync_tmpfile_name); - return(false); - } - } - - msg("Starting %s non-InnoDB tables and files", - prep_mode ? "prep copy of" : "to backup"); - + msg("Starting to backup non-InnoDB tables and files"); datadir_node_init(&node); it = datadir_iter_new(from); - while (datadir_iter_next(it, &node)) { - if (!node.is_empty_dir) { - if (opt_rsync) { - ret = datafile_rsync_backup(node.filepath, - !prep_mode, rsync_tmpfile); - } else { - ret = datafile_copy_backup(ds_data, node.filepath, 1); - } + ret = datafile_copy_backup(ds_data, node.filepath, 1); if (!ret) { msg("Failed to copy file %s", node.filepath); goto out; } - } else if (!prep_mode) { + } else { /* backup fake file into empty directory */ char path[FN_REFLEN]; - snprintf(path, sizeof(path), - "%s/db.opt", node.filepath); - if (!(ret = ds_data->backup_file_printf( - trim_dotslash(path), "%s", ""))) { + snprintf(path, sizeof(path), "%s/db.opt", node.filepath); + if (!(ret = ds_data->backup_file_printf(trim_dotslash(path), "%s", ""))) { msg("Failed to create file %s", path); goto out; } } } - - if (opt_rsync) { - std::stringstream cmd; - int err; - - if (buffer_pool_filename && file_exists(buffer_pool_filename)) { - fprintf(rsync_tmpfile, "%s\n", buffer_pool_filename); - rsync_list.insert(buffer_pool_filename); - } - if (file_exists("ib_lru_dump")) { - fprintf(rsync_tmpfile, "%s\n", "ib_lru_dump"); - rsync_list.insert("ib_lru_dump"); - } - - fclose(rsync_tmpfile); - rsync_tmpfile = NULL; - - cmd << "rsync -t . --files-from=" << rsync_tmpfile_name - << " " << xtrabackup_target_dir; - - msg("Starting rsync as: %s", cmd.str().c_str()); - if ((err = system(cmd.str().c_str()) && !prep_mode) != 0) { - msg("Error: rsync failed with error code %d", err); - ret = false; - goto out; - } - msg("rsync finished successfully."); - - if (!prep_mode && !opt_no_lock) { - char path[FN_REFLEN]; - char dst_path[FN_REFLEN]; - char *newline; - - /* Remove files that have been removed between first and - second passes. Cannot use "rsync --delete" because it - does not work with --files-from. */ - snprintf(rsync_tmpfile_name, sizeof(rsync_tmpfile_name), - "%s/%s", opt_mysql_tmpdir, - "xtrabackup_rsyncfiles_pass1"); - - rsync_tmpfile = fopen(rsync_tmpfile_name, "r"); - if (rsync_tmpfile == NULL) { - msg("Error: can't open file %s", - rsync_tmpfile_name); - ret = false; - goto out; - } - - while (fgets(path, sizeof(path), rsync_tmpfile)) { - - newline = strchr(path, '\n'); - if (newline) { - *newline = 0; - } - if (rsync_list.count(path) < 1) { - snprintf(dst_path, sizeof(dst_path), - "%s/%s", xtrabackup_target_dir, - path); - msg("Removing %s", dst_path); - unlink(dst_path); - } - } - - fclose(rsync_tmpfile); - rsync_tmpfile = NULL; - } - } - - msg("Finished %s non-InnoDB tables and files", - prep_mode ? "a prep copy of" : "backing up"); - + msg("Finished backing up non-InnoDB tables and files"); out: datadir_iter_free(it); datadir_node_free(&node); - - if (rsync_tmpfile != NULL) { - fclose(rsync_tmpfile); - } - return(ret); } - -lsn_t get_current_lsn(MYSQL *connection) -{ - static const char lsn_prefix[] = "\nLog sequence number "; - lsn_t lsn = 0; - if (MYSQL_RES *res = xb_mysql_query(connection, - "SHOW ENGINE INNODB STATUS", - true, false)) { - if (MYSQL_ROW row = mysql_fetch_row(res)) { - const char *p= strstr(row[2], lsn_prefix); - DBUG_ASSERT(p); - if (p) { - p += sizeof lsn_prefix - 1; - lsn = lsn_t(strtoll(p, NULL, 10)); - } - } - mysql_free_result(res); - } - return lsn; -} - lsn_t server_lsn_after_lock; extern void backup_wait_for_lsn(lsn_t lsn); -/** Start --backup */ -bool backup_start(ds_ctxt *ds_data, ds_ctxt *ds_meta, - CorruptedPages &corrupted_pages) -{ - if (!opt_no_lock) { - if (opt_safe_slave_backup) { - if (!wait_for_safe_slave(mysql_connection)) { - return(false); - } - } - - if (!backup_files(ds_data, fil_path_to_mysql_datadir, true)) { - return(false); - } - - history_lock_time = time(NULL); - - if (!lock_tables(mysql_connection)) { - return(false); - } - server_lsn_after_lock = get_current_lsn(mysql_connection); - } - - if (!backup_files(ds_data, fil_path_to_mysql_datadir, false)) { - return(false); - } - - if (!backup_files_from_datadir(ds_data, fil_path_to_mysql_datadir, - "aws-kms-key") || - !backup_files_from_datadir(ds_data, - aria_log_dir_path, - "aria_log")) { - return false; - } - if (has_rocksdb_plugin()) { - rocksdb_create_checkpoint(); - } - - msg("Waiting for log copy thread to read lsn %llu", (ulonglong)server_lsn_after_lock); - backup_wait_for_lsn(server_lsn_after_lock); - DBUG_EXECUTE_FOR_KEY("sleep_after_waiting_for_lsn", {}, - { - ulong milliseconds = strtoul(dbug_val, NULL, 10); - msg("sleep_after_waiting_for_lsn"); - my_sleep(milliseconds*1000UL); - }); - - corrupted_pages.backup_fix_ddl(ds_data, ds_meta); - - // There is no need to stop slave thread before coping non-Innodb data when - // --no-lock option is used because --no-lock option requires that no DDL or - // DML to non-transaction tables can occur. - if (opt_no_lock) { - if (opt_safe_slave_backup) { - if (!wait_for_safe_slave(mysql_connection)) { - return(false); - } - } - } - - if (opt_slave_info) { - lock_binlog_maybe(mysql_connection); - - if (!write_slave_info(ds_data, mysql_connection)) { - return(false); - } - } - - /* The only reason why Galera/binlog info is written before - wait_for_ibbackup_log_copy_finish() is that after that call the xtrabackup - binary will start streamig a temporary copy of REDO log to stdout and - thus, any streaming from innobackupex would interfere. The only way to - avoid that is to have a single process, i.e. merge innobackupex and - xtrabackup. */ - if (opt_galera_info) { - if (!write_galera_info(ds_data, mysql_connection)) { - return(false); - } - } - - if (opt_binlog_info == BINLOG_INFO_ON) { - - lock_binlog_maybe(mysql_connection); - write_binlog_info(ds_data, mysql_connection); - } - - if (!opt_no_lock) { - msg("Executing FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS..."); - xb_mysql_query(mysql_connection, - "FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS", false); - } - - return(true); -} - -/** Release resources after backup_start() */ +/** Release resources after backup_files() */ void backup_release() { - /* release all locks */ - if (!opt_no_lock) { - unlock_all(mysql_connection); - history_lock_time = 0; - } else { - history_lock_time = time(NULL) - history_lock_time; - } - if (opt_lock_ddl_per_table) { mdl_unlock_all(); } @@ -1530,11 +1372,11 @@ void backup_release() static const char *default_buffer_pool_file = "ib_buffer_pool"; -/** Finish after backup_start() and backup_release() */ +/** Finish after backup_files() and backup_release() */ bool backup_finish(ds_ctxt *ds_data) { /* Copy buffer pool dump or LRU dump */ - if (!opt_rsync && opt_galera_info) { + if (opt_galera_info) { if (buffer_pool_filename && file_exists(buffer_pool_filename)) { ds_data->copy_file(buffer_pool_filename, default_buffer_pool_file, 0); } @@ -1893,8 +1735,6 @@ copy_back() return(false); } - srv_max_n_threads = 1000; - /* copy undo tablespaces */ Copy_back_dst_dir dst_dir_buf; @@ -1922,7 +1762,8 @@ copy_back() dst_dir = dst_dir_buf.make(srv_log_group_home_dir); - /* --backup generates a single ib_logfile0, which we must copy. */ + /* --backup generates a single LOG_FILE_NAME, which we must copy + if it exists. */ ds_tmp = ds_create(dst_dir, DS_TYPE_LOCAL); if (!(ret = copy_or_move_file(ds_tmp, LOG_FILE_NAME, LOG_FILE_NAME, @@ -2155,8 +1996,6 @@ decrypt_decompress() bool ret; datadir_iter_t *it = NULL; - srv_max_n_threads = 1000; - /* cd to backup directory */ if (my_setwd(xtrabackup_target_dir, MYF(MY_WME))) { @@ -2169,8 +2008,6 @@ decrypt_decompress() it = datadir_iter_new(".", false); - ut_a(xtrabackup_parallel >= 0); - ret = run_data_threads(it, decrypt_decompress_thread_func, xtrabackup_parallel ? xtrabackup_parallel : 1); @@ -2192,9 +2029,9 @@ decrypt_decompress() Do not copy the Innodb files (ibdata1, redo log files), as this is done in a separate step. */ -static bool backup_files_from_datadir(ds_ctxt_t *ds_data, - const char *dir_path, - const char *prefix) +bool backup_files_from_datadir(ds_ctxt_t *ds_data, + const char *dir_path, + const char *prefix) { os_file_dir_t dir = os_file_opendir(dir_path); if (dir == IF_WIN(INVALID_HANDLE_VALUE, nullptr)) return false; @@ -2218,10 +2055,6 @@ static bool backup_files_from_datadir(ds_ctxt_t *ds_data, pname = info.name; if (!starts_with(pname, prefix)) - /* For ES exchange the above line with the following code: - (!xtrabackup_prepare || !xtrabackup_incremental_dir || - !starts_with(pname, "aria_log"))) - */ continue; if (xtrabackup_prepare && xtrabackup_incremental_dir && @@ -2244,7 +2077,7 @@ static int rocksdb_remove_checkpoint_directory() return 0; } -static bool has_rocksdb_plugin() +bool has_rocksdb_plugin() { static bool first_time = true; static bool has_plugin= false; @@ -2390,7 +2223,7 @@ static void rocksdb_unlock_checkpoint() #define MARIADB_CHECKPOINT_DIR "mariabackup-checkpoint" static char rocksdb_checkpoint_dir[FN_REFLEN]; -static void rocksdb_create_checkpoint() +void rocksdb_create_checkpoint() { MYSQL_RES *result = xb_mysql_query(mysql_connection, "SELECT @@rocksdb_datadir,@@datadir", true, true); MYSQL_ROW row = mysql_fetch_row(result); @@ -2470,3 +2303,39 @@ static void rocksdb_copy_back(ds_ctxt *ds_data) { mkdirp(rocksdb_home_dir, 0777, MYF(0)); ds_data->copy_or_move_dir(ROCKSDB_BACKUP_DIR, rocksdb_home_dir, xtrabackup_copy_back, xtrabackup_copy_back); } + +void foreach_file_in_db_dirs( + const char *dir_path, std::function func) { + DBUG_ASSERT(dir_path); + + datadir_iter_t *it; + datadir_node_t node; + + datadir_node_init(&node); + it = datadir_iter_new(dir_path); + + while (datadir_iter_next(it, &node)) + if (!node.is_empty_dir && !func(node.filepath)) + break; + + datadir_iter_free(it); + datadir_node_free(&node); +} + +void foreach_file_in_datadir( + const char *dir_path, std::function func) +{ + DBUG_ASSERT(dir_path); + os_file_dir_t dir = os_file_opendir(dir_path); + os_file_stat_t info; + while (os_file_readdir_next_file(dir_path, dir, &info) == 0) { + if (info.type != OS_FILE_TYPE_FILE) + continue; + const char *pname = strrchr(info.name, IF_WIN('\\', '/')); + if (!pname) + pname = info.name; + if (!func(pname)) + break; + } + os_file_closedir(dir); +} diff --git a/extra/mariabackup/backup_copy.h b/extra/mariabackup/backup_copy.h index b5aaf312..409e7839 100644 --- a/extra/mariabackup/backup_copy.h +++ b/extra/mariabackup/backup_copy.h @@ -2,6 +2,7 @@ #ifndef XTRABACKUP_BACKUP_COPY_H #define XTRABACKUP_BACKUP_COPY_H +#include #include #include #include "datasink.h" @@ -21,11 +22,10 @@ bool equal_paths(const char *first, const char *second); /** Start --backup */ -bool backup_start(ds_ctxt *ds_data, ds_ctxt *ds_meta, - CorruptedPages &corrupted_pages); -/** Release resources after backup_start() */ +bool backup_files(ds_ctxt *ds_data, const char *from); +/** Release resources after backup_files() */ void backup_release(); -/** Finish after backup_start() and backup_release() */ +/** Finish after backup_files() and backup_release() */ bool backup_finish(ds_ctxt *ds_data); bool apply_log_finish(); @@ -38,7 +38,25 @@ is_path_separator(char); bool directory_exists(const char *dir, bool create); -lsn_t -get_current_lsn(MYSQL *connection); - +bool has_rocksdb_plugin(); +void rocksdb_create_checkpoint(); +void foreach_file_in_db_dirs( + const char *dir_path, std::function func); +void foreach_file_in_datadir( + const char *dir_path, std::function func); +bool ends_with(const char *str, const char *suffix); +bool starts_with(const char *str, const char *prefix); +void parse_db_table_from_file_path( + const char *filepath, char *dbname, char *tablename); +const char *trim_dotslash(const char *path); +bool backup_files_from_datadir(ds_ctxt_t *ds_data, + const char *dir_path, + const char *prefix); + +bool is_system_table(const char *dbname, const char *tablename); +std::unique_ptr> + find_files(const char *dir_path, const char *prefix, const char *suffix); +bool file_exists(const char *filename); +bool +filename_matches(const char *filename, const char **ext_list); #endif diff --git a/extra/mariabackup/backup_debug.h b/extra/mariabackup/backup_debug.h index 777b4f4a..9286bc7b 100644 --- a/extra/mariabackup/backup_debug.h +++ b/extra/mariabackup/backup_debug.h @@ -1,5 +1,6 @@ #pragma once #include "my_dbug.h" + #ifndef DBUG_OFF char *dbug_mariabackup_get_val(const char *event, fil_space_t::name_type key); /* @@ -14,11 +15,21 @@ To use this facility, you need to for the variable) 3. start mariabackup with --dbug=+d,debug_mariabackup_events */ -#define DBUG_EXECUTE_FOR_KEY(EVENT, KEY, CODE) \ - DBUG_EXECUTE_IF("mariabackup_inject_code", \ - { char *dbug_val= dbug_mariabackup_get_val(EVENT, KEY); \ - if (dbug_val) CODE }) +extern void dbug_mariabackup_event( + const char *event, const fil_space_t::name_type key, bool need_lock); +#define DBUG_MARIABACKUP_EVENT(A, B) \ + DBUG_EXECUTE_IF("mariabackup_events", \ + dbug_mariabackup_event(A,B,false);); +#define DBUG_MARIABACKUP_EVENT_LOCK(A, B) \ + DBUG_EXECUTE_IF("mariabackup_events", \ + dbug_mariabackup_event(A,B, true);); +#define DBUG_EXECUTE_FOR_KEY(EVENT, KEY, CODE) \ + DBUG_EXECUTE_IF("mariabackup_inject_code", {\ + char *dbug_val = dbug_mariabackup_get_val(EVENT, KEY); \ + if (dbug_val && *dbug_val) CODE \ + }) #else +#define DBUG_MARIABACKUP_EVENT(A,B) +#define DBUG_MARIABACKUP_EVENT_LOCK(A,B) #define DBUG_EXECUTE_FOR_KEY(EVENT, KEY, CODE) #endif - diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc index c2f15da4..2aad6004 100644 --- a/extra/mariabackup/backup_mysql.cc +++ b/extra/mariabackup/backup_mysql.cc @@ -47,6 +47,12 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA #include #include #include +#ifdef HAVE_PWD_H +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#include +#endif #include "common.h" #include "xtrabackup.h" #include "srv0srv.h" @@ -54,19 +60,19 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA #include "backup_copy.h" #include "backup_mysql.h" #include "mysqld.h" -#include "xb_plugin.h" +#include "encryption_plugin.h" #include #include #include "page0zip.h" +#include "backup_debug.h" char *tool_name; -char tool_args[2048]; +char tool_args[8192]; ulong mysql_server_version; /* server capabilities */ bool have_changed_page_bitmaps = false; -bool have_backup_locks = false; bool have_lock_wait_timeout = false; bool have_galera_enabled = false; bool have_multi_threaded_slave = false; @@ -92,11 +98,54 @@ MYSQL *mysql_connection; extern my_bool opt_ssl_verify_server_cert, opt_use_ssl; + +/* + get_os_user() + Ressemles read_user_name() from libmariadb/libmariadb/mariadb_lib.c. +*/ + +#if !defined(_WIN32) + +#if defined(HAVE_GETPWUID) && defined(NO_GETPWUID_DECL) +struct passwd *getpwuid(uid_t); +char* getlogin(void); +#endif + +static const char *get_os_user() // Posix +{ + if (!geteuid()) + return "root"; +#ifdef HAVE_GETPWUID + struct passwd *pw; + const char *str; + if ((pw= getpwuid(geteuid())) != NULL) + return pw->pw_name; + if ((str= getlogin()) != NULL) + return str; +#endif + if ((str= getenv("USER")) || + (str= getenv("LOGNAME")) || + (str= getenv("LOGIN"))) + return str; + return NULL; +} + +#else + +static const char *get_os_user() // Windows +{ + return getenv("USERNAME"); +} + +#endif // _WIN32 + + MYSQL * xb_mysql_connect() { MYSQL *connection = mysql_init(NULL); char mysql_port_str[std::numeric_limits::digits10 + 3]; + const char *user= opt_user ? opt_user : get_os_user(); sprintf(mysql_port_str, "%d", opt_port); @@ -126,7 +175,7 @@ xb_mysql_connect() msg("Connecting to MariaDB server host: %s, user: %s, password: %s, " "port: %s, socket: %s", opt_host ? opt_host : "localhost", - opt_user ? opt_user : "not set", + user ? user : "not set", opt_password ? "set" : "not set", opt_port != 0 ? mysql_port_str : "not set", opt_socket ? opt_socket : "not set"); @@ -147,7 +196,7 @@ xb_mysql_connect() if (!mysql_real_connect(connection, opt_host ? opt_host : "localhost", - opt_user, + user, opt_password, "" /*database*/, opt_port, opt_socket, 0)) { @@ -203,13 +252,14 @@ struct mysql_variable { static -void +uint read_mysql_variables(MYSQL *connection, const char *query, mysql_variable *vars, bool vertical_result) { MYSQL_RES *mysql_result; MYSQL_ROW row; mysql_variable *var; + uint n_values=0; mysql_result = xb_mysql_query(connection, query, true); @@ -223,6 +273,7 @@ read_mysql_variables(MYSQL *connection, const char *query, mysql_variable *vars, if (strcmp(var->name, name) == 0 && value != NULL) { *(var->value) = strdup(value); + n_values++; } } } @@ -239,6 +290,7 @@ read_mysql_variables(MYSQL *connection, const char *query, mysql_variable *vars, if (strcmp(var->name, name) == 0 && value != NULL) { *(var->value) = strdup(value); + n_values++; } } ++i; @@ -247,6 +299,7 @@ read_mysql_variables(MYSQL *connection, const char *query, mysql_variable *vars, } mysql_free_result(mysql_result); + return n_values; } @@ -311,7 +364,6 @@ bool get_mysql_vars(MYSQL *connection) { char *gtid_mode_var= NULL; char *version_var= NULL; - char *have_backup_locks_var= NULL; char *log_bin_var= NULL; char *lock_wait_timeout_var= NULL; char *wsrep_on_var= NULL; @@ -336,7 +388,6 @@ bool get_mysql_vars(MYSQL *connection) bool ret= true; mysql_variable mysql_vars[]= { - {"have_backup_locks", &have_backup_locks_var}, {"log_bin", &log_bin_var}, {"lock_wait_timeout", &lock_wait_timeout_var}, {"gtid_mode", >id_mode_var}, @@ -361,11 +412,6 @@ bool get_mysql_vars(MYSQL *connection) read_mysql_variables(connection, "SHOW VARIABLES", mysql_vars, true); - if (have_backup_locks_var != NULL && !opt_no_backup_locks) - { - have_backup_locks= true; - } - if (opt_binlog_info == BINLOG_INFO_AUTO) { if (log_bin_var != NULL && !strcmp(log_bin_var, "ON")) @@ -512,24 +558,6 @@ Query the server to find out what backup capabilities it supports. bool detect_mysql_capabilities_for_backup() { - const char *query = "SELECT 'INNODB_CHANGED_PAGES', COUNT(*) FROM " - "INFORMATION_SCHEMA.PLUGINS " - "WHERE PLUGIN_NAME LIKE 'INNODB_CHANGED_PAGES'"; - char *innodb_changed_pages = NULL; - mysql_variable vars[] = { - {"INNODB_CHANGED_PAGES", &innodb_changed_pages}, {NULL, NULL}}; - - if (xtrabackup_incremental) { - - read_mysql_variables(mysql_connection, query, vars, true); - - ut_ad(innodb_changed_pages != NULL); - - have_changed_page_bitmaps = (atoi(innodb_changed_pages) == 1); - - free_mysql_variables(vars); - } - /* do some sanity checks */ if (opt_galera_info && !have_galera_enabled) { msg("--galera-info is specified on the command " @@ -837,11 +865,11 @@ static void stop_query_killer() /*********************************************************************//** -Function acquires either a backup tables lock, if supported -by the server, or a global read lock (FLUSH TABLES WITH READ LOCK) -otherwise. +Function acquires backup locks @returns true if lock acquired */ -bool lock_tables(MYSQL *connection) + +bool +lock_for_backup_stage_start(MYSQL *connection) { if (have_lock_wait_timeout || opt_lock_wait_timeout) { @@ -854,12 +882,6 @@ bool lock_tables(MYSQL *connection) xb_mysql_query(connection, buf, false); } - if (have_backup_locks) - { - msg("Executing LOCK TABLES FOR BACKUP..."); - xb_mysql_query(connection, "LOCK TABLES FOR BACKUP", false); - return (true); - } if (opt_lock_wait_timeout) { @@ -884,8 +906,6 @@ bool lock_tables(MYSQL *connection) xb_mysql_query(connection, "BACKUP STAGE START", true); DBUG_MARIABACKUP_EVENT("after_backup_stage_start", {}); - xb_mysql_query(connection, "BACKUP STAGE BLOCK_COMMIT", true); - DBUG_MARIABACKUP_EVENT("after_backup_stage_block_commit", {}); /* Set the maximum supported session value for lock_wait_timeout to prevent unnecessary timeouts when the global value is changed from the default */ @@ -901,24 +921,68 @@ bool lock_tables(MYSQL *connection) return (true); } -/*********************************************************************//** -If backup locks are used, execute LOCK BINLOG FOR BACKUP provided that we are -not in the --no-lock mode and the lock has not been acquired already. -@returns true if lock acquired */ bool -lock_binlog_maybe(MYSQL *connection) -{ - if (have_backup_locks && !opt_no_lock && !binlog_locked) { - msg("Executing LOCK BINLOG FOR BACKUP..."); - xb_mysql_query(connection, "LOCK BINLOG FOR BACKUP", false); - binlog_locked = true; +lock_for_backup_stage_flush(MYSQL *connection) { + if (opt_kill_long_queries_timeout) { + start_query_killer(); + } + xb_mysql_query(connection, "BACKUP STAGE FLUSH", true); + if (opt_kill_long_queries_timeout) { + stop_query_killer(); + } + return true; +} - return(true); +bool +lock_for_backup_stage_block_ddl(MYSQL *connection) { + if (opt_kill_long_queries_timeout) { + start_query_killer(); + } + xb_mysql_query(connection, "BACKUP STAGE BLOCK_DDL", true); + DBUG_MARIABACKUP_EVENT("after_backup_stage_block_ddl", {}); + if (opt_kill_long_queries_timeout) { + stop_query_killer(); } + return true; +} - return(false); +bool +lock_for_backup_stage_commit(MYSQL *connection) { + if (opt_kill_long_queries_timeout) { + start_query_killer(); + } + xb_mysql_query(connection, "BACKUP STAGE BLOCK_COMMIT", true); + DBUG_MARIABACKUP_EVENT("after_backup_stage_block_commit", {}); + if (opt_kill_long_queries_timeout) { + stop_query_killer(); + } + return true; } +bool backup_lock(MYSQL *con, const char *table_name) { + static const std::string backup_lock_prefix("BACKUP LOCK "); + std::string backup_lock_query = backup_lock_prefix + table_name; + xb_mysql_query(con, backup_lock_query.c_str(), true); + return true; +} + +bool backup_unlock(MYSQL *con) { + xb_mysql_query(con, "BACKUP UNLOCK", true); + return true; +} + +std::unordered_set +get_tables_in_use(MYSQL *con) { + std::unordered_set result; + MYSQL_RES *q_res = + xb_mysql_query(con, "SHOW OPEN TABLES WHERE In_use = 1", true); + while (MYSQL_ROW row = mysql_fetch_row(q_res)) { + auto tk = table_key(row[0], row[1]); + msg("Table %s is in use", tk.c_str()); + result.insert(std::move(tk)); + } + return result; +} /*********************************************************************//** Releases either global read lock acquired with FTWRL and the binlog @@ -1353,77 +1417,103 @@ write_slave_info(ds_ctxt *datasink, MYSQL *connection) /*********************************************************************//** -Retrieves MySQL Galera and -saves it in a file. It also prints it to stdout. */ +Retrieves MySQL Galera and saves it in a file. It also prints it to stdout. + +We should create xtrabackup_galelera_info file even when backup locks +are used because donor's wsrep_gtid_domain_id is needed later in joiner. +Note that at this stage wsrep_local_state_uuid and wsrep_last_committed +are inconsistent but they are not used in joiner. Joiner will rewrite this file +at mariabackup --prepare phase and thus there is extra file donor_galera_info. +Information is needed to maitain wsrep_gtid_domain_id and gtid_binlog_pos +same across the cluster. If joiner node have different wsrep_gtid_domain_id +we should still receive effective domain id from the donor node, +and use it. +*/ bool write_galera_info(ds_ctxt *datasink, MYSQL *connection) { - char *state_uuid = NULL, *state_uuid55 = NULL; - char *last_committed = NULL, *last_committed55 = NULL; - char *domain_id = NULL, *domain_id55 = NULL; - bool result; - - mysql_variable status[] = { - {"Wsrep_local_state_uuid", &state_uuid}, - {"wsrep_local_state_uuid", &state_uuid55}, - {"Wsrep_last_committed", &last_committed}, - {"wsrep_last_committed", &last_committed55}, - {NULL, NULL} - }; + char *state_uuid = NULL, *state_uuid55 = NULL; + char *last_committed = NULL, *last_committed55 = NULL; + char *domain_id = NULL, *domain_id55 = NULL; + bool result=true; + uint n_values=0; + char *wsrep_on = NULL, *wsrep_on55 = NULL; + + mysql_variable vars[] = { + {"Wsrep_on", &wsrep_on}, + {"wsrep_on", &wsrep_on55}, + {NULL, NULL} + }; + + mysql_variable status[] = { + {"Wsrep_local_state_uuid", &state_uuid}, + {"wsrep_local_state_uuid", &state_uuid55}, + {"Wsrep_last_committed", &last_committed}, + {"wsrep_last_committed", &last_committed55}, + {NULL, NULL} + }; + + mysql_variable value[] = { + {"Wsrep_gtid_domain_id", &domain_id}, + {"wsrep_gtid_domain_id", &domain_id55}, + {NULL, NULL} + }; + + n_values= read_mysql_variables(connection, "SHOW VARIABLES", vars, true); + + if (n_values == 0 || (wsrep_on == NULL && wsrep_on55 == NULL)) + { + msg("Server is not Galera node thus --galera-info does not " + "have any effect."); + result = true; + goto cleanup; + } - mysql_variable value[] = { - {"Wsrep_gtid_domain_id", &domain_id}, - {"wsrep_gtid_domain_id", &domain_id55}, - {NULL, NULL} - }; + read_mysql_variables(connection, "SHOW STATUS", status, true); - /* When backup locks are supported by the server, we should skip - creating xtrabackup_galera_info file on the backup stage, because - wsrep_local_state_uuid and wsrep_last_committed will be inconsistent - without blocking commits. The state file will be created on the prepare - stage using the WSREP recovery procedure. */ - if (have_backup_locks) { - return(true); - } + if ((state_uuid == NULL && state_uuid55 == NULL) + || (last_committed == NULL && last_committed55 == NULL)) + { + msg("Warning: failed to get master wsrep state from SHOW STATUS."); + result = true; + goto cleanup; + } - read_mysql_variables(connection, "SHOW STATUS", status, true); + n_values= read_mysql_variables(connection, "SHOW VARIABLES LIKE 'wsrep%'", value, true); - if ((state_uuid == NULL && state_uuid55 == NULL) - || (last_committed == NULL && last_committed55 == NULL)) { - msg("Warning: failed to get master wsrep state from SHOW STATUS."); - result = true; - goto cleanup; - } + if (n_values == 0 || (domain_id == NULL && domain_id55 == NULL)) + { + msg("Warning: failed to get master wsrep state from SHOW VARIABLES."); + result = true; + goto cleanup; + } - read_mysql_variables(connection, "SHOW VARIABLES LIKE 'wsrep%'", value, true); + result= datasink->backup_file_printf(XTRABACKUP_GALERA_INFO, + "%s:%s %s\n", state_uuid ? state_uuid : state_uuid55, + last_committed ? last_committed : last_committed55, + domain_id ? domain_id : domain_id55); - if (domain_id == NULL && domain_id55 == NULL) { - msg("Warning: failed to get master wsrep state from SHOW VARIABLES."); - result = true; - goto cleanup; - } + if (result) + { + result= datasink->backup_file_printf(XTRABACKUP_DONOR_GALERA_INFO, + "%s:%s %s\n", state_uuid ? state_uuid : state_uuid55, + last_committed ? last_committed : last_committed55, + domain_id ? domain_id : domain_id55); + } - result = datasink->backup_file_printf(XTRABACKUP_GALERA_INFO, - "%s:%s %s\n", state_uuid ? state_uuid : state_uuid55, - last_committed ? last_committed : last_committed55, - domain_id ? domain_id : domain_id55); + if (result) + write_current_binlog_file(datasink, connection); - if (result) - { - result= datasink->backup_file_printf(XTRABACKUP_DONOR_GALERA_INFO, - "%s:%s %s\n", state_uuid ? state_uuid : state_uuid55, - last_committed ? last_committed : last_committed55, - domain_id ? domain_id : domain_id55); - } - if (result) - { - write_current_binlog_file(datasink, connection); - } + if (result) + msg("Writing Galera info succeeded with %s:%s %s", + state_uuid ? state_uuid : state_uuid55, + last_committed ? last_committed : last_committed55, + domain_id ? domain_id : domain_id55); cleanup: - free_mysql_variables(status); + free_mysql_variables(status); - return(result); + return(result); } @@ -1466,8 +1556,6 @@ write_current_binlog_file(ds_ctxt *datasink, MYSQL *connection) if (gtid_exists) { size_t log_bin_dir_length; - lock_binlog_maybe(connection); - xb_mysql_query(connection, "FLUSH BINARY LOGS", false); read_mysql_variables(connection, "SHOW MASTER STATUS", @@ -1826,13 +1914,13 @@ bool write_backup_config_file(ds_ctxt *datasink) srv_log_file_size, srv_page_size, srv_undo_dir, - srv_undo_tablespaces, + (uint) srv_undo_tablespaces, page_zip_level, innobase_buffer_pool_filename ? "innodb_buffer_pool_filename=" : "", innobase_buffer_pool_filename ? innobase_buffer_pool_filename : "", - xb_plugin_get_config()); + encryption_plugin_get_config()); return rc; } @@ -1851,9 +1939,11 @@ char *make_argv(char *buf, size_t len, int argc, char **argv) if (strncmp(*argv, "--password", strlen("--password")) == 0) { arg = "--password=..."; } - left-= snprintf(buf + len - left, left, + uint l= snprintf(buf + len - left, left, "%s%c", arg, argc > 1 ? ' ' : 0); ++argv; --argc; + if (l < left) + left-= l; } return buf; @@ -1882,18 +1972,6 @@ select_history() return(true); } -bool -flush_changed_page_bitmaps() -{ - if (xtrabackup_incremental && have_changed_page_bitmaps && - !xtrabackup_incremental_force_scan) { - xb_mysql_query(mysql_connection, - "FLUSH NO_WRITE_TO_BINLOG CHANGED_PAGE_BITMAPS", false); - } - return(true); -} - - /*********************************************************************//** Deallocate memory, disconnect from server, etc. @return true on success. */ @@ -1969,3 +2047,23 @@ mdl_unlock_all() mysql_close(mdl_con); spaceid_to_tablename.clear(); } + +ulonglong get_current_lsn(MYSQL *connection) +{ + static const char lsn_prefix[] = "\nLog sequence number "; + ulonglong lsn = 0; + if (MYSQL_RES *res = xb_mysql_query(connection, + "SHOW ENGINE INNODB STATUS", + true, false)) { + if (MYSQL_ROW row = mysql_fetch_row(res)) { + const char *p= strstr(row[2], lsn_prefix); + DBUG_ASSERT(p); + if (p) { + p += sizeof lsn_prefix - 1; + lsn = lsn_t(strtoll(p, NULL, 10)); + } + } + mysql_free_result(res); + } + return lsn; +} diff --git a/extra/mariabackup/backup_mysql.h b/extra/mariabackup/backup_mysql.h index 4b08da0b..c87efd21 100644 --- a/extra/mariabackup/backup_mysql.h +++ b/extra/mariabackup/backup_mysql.h @@ -2,13 +2,15 @@ #define XTRABACKUP_BACKUP_MYSQL_H #include +#include +#include +#include "datasink.h" /* MariaDB version */ extern ulong mysql_server_version; /* server capabilities */ extern bool have_changed_page_bitmaps; -extern bool have_backup_locks; extern bool have_lock_wait_timeout; extern bool have_galera_enabled; extern bool have_multi_threaded_slave; @@ -35,9 +37,6 @@ capture_tool_command(int argc, char **argv); bool select_history(); -bool -flush_changed_page_bitmaps(); - void backup_cleanup(); @@ -75,7 +74,21 @@ bool lock_binlog_maybe(MYSQL *connection); bool -lock_tables(MYSQL *connection); +lock_for_backup_stage_start(MYSQL *connection); + +bool +lock_for_backup_stage_flush(MYSQL *connection); + +bool +lock_for_backup_stage_block_ddl(MYSQL *connection); + +bool +lock_for_backup_stage_commit(MYSQL *connection); + +bool backup_lock(MYSQL *con, const char *table_name); +bool backup_unlock(MYSQL *con); + +std::unordered_set get_tables_in_use(MYSQL *con); bool wait_for_safe_slave(MYSQL *connection); @@ -86,5 +99,6 @@ write_galera_info(ds_ctxt *datasink, MYSQL *connection); bool write_slave_info(ds_ctxt *datasink, MYSQL *connection); +ulonglong get_current_lsn(MYSQL *connection); #endif diff --git a/extra/mariabackup/changed_page_bitmap.cc b/extra/mariabackup/changed_page_bitmap.cc deleted file mode 100644 index 39a07a25..00000000 --- a/extra/mariabackup/changed_page_bitmap.cc +++ /dev/null @@ -1,1040 +0,0 @@ -/****************************************************** -XtraBackup: hot backup tool for InnoDB -(c) 2009-2012 Percona Inc. -Originally Created 3/3/2009 Yasufumi Kinoshita -Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, -Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA - -*******************************************************/ - -/* Changed page bitmap implementation */ - -#include "changed_page_bitmap.h" - -#include "common.h" -#include "xtrabackup.h" -#include "srv0srv.h" - -/* TODO: copy-pasted shared definitions from the XtraDB bitmap write code. -Remove these on the first opportunity, i.e. single-binary XtraBackup. */ - -/* log0online.h */ - -/** Single bitmap file information */ -struct log_online_bitmap_file_t { - char name[FN_REFLEN]; /*!< Name with full path */ - pfs_os_file_t file; /*!< Handle to opened file */ - ib_uint64_t size; /*!< Size of the file */ - ib_uint64_t offset; /*!< Offset of the next read, - or count of already-read bytes - */ -}; - -/** A set of bitmap files containing some LSN range */ -struct log_online_bitmap_file_range_t { - size_t count; /*!< Number of files */ - /*!< Dynamically-allocated array of info about individual files */ - struct files_t { - char name[FN_REFLEN];/*!< Name of a file */ - lsn_t start_lsn; /*!< Starting LSN of data in this - file */ - ulong seq_num; /*!< Sequence number of this file */ - } *files; -}; - -/* log0online.c */ - -/** File name stem for bitmap files. */ -static const char* bmp_file_name_stem = "ib_modified_log_"; - -/** The bitmap file block size in bytes. All writes will be multiples of this. - */ -enum { - MODIFIED_PAGE_BLOCK_SIZE = 4096 -}; - -/** Offsets in a file bitmap block */ -enum { - MODIFIED_PAGE_IS_LAST_BLOCK = 0,/* 1 if last block in the current - write, 0 otherwise. */ - MODIFIED_PAGE_START_LSN = 4, /* The starting tracked LSN of this and - other blocks in the same write */ - MODIFIED_PAGE_END_LSN = 12, /* The ending tracked LSN of this and - other blocks in the same write */ - MODIFIED_PAGE_SPACE_ID = 20, /* The space ID of tracked pages in - this block */ - MODIFIED_PAGE_1ST_PAGE_ID = 24, /* The page ID of the first tracked - page in this block */ - MODIFIED_PAGE_BLOCK_UNUSED_1 = 28,/* Unused in order to align the start - of bitmap at 8 byte boundary */ - MODIFIED_PAGE_BLOCK_BITMAP = 32,/* Start of the bitmap itself */ - MODIFIED_PAGE_BLOCK_UNUSED_2 = MODIFIED_PAGE_BLOCK_SIZE - 8, - /* Unused in order to align the end of - bitmap at 8 byte boundary */ - MODIFIED_PAGE_BLOCK_CHECKSUM = MODIFIED_PAGE_BLOCK_SIZE - 4 - /* The checksum of the current block */ -}; - -/** Length of the bitmap data in a block */ -enum { MODIFIED_PAGE_BLOCK_BITMAP_LEN - = MODIFIED_PAGE_BLOCK_UNUSED_2 - MODIFIED_PAGE_BLOCK_BITMAP }; - -/** Length of the bitmap data in a block in page ids */ -enum { MODIFIED_PAGE_BLOCK_ID_COUNT = MODIFIED_PAGE_BLOCK_BITMAP_LEN * 8 }; - -typedef ib_uint64_t bitmap_word_t; - -/****************************************************************//** -Calculate a bitmap block checksum. Algorithm borrowed from -log_block_calc_checksum. -@return checksum */ -UNIV_INLINE -ulint -log_online_calc_checksum( -/*=====================*/ - const byte* block); /*! p2 -*/ -static -int -log_online_compare_bmp_keys( -/*========================*/ - const void* p1, /*! k2_start_page ? 1 : 0; - } - return k1_space < k2_space ? -1 : 1; -} - -/****************************************************************//** -Calculate a bitmap block checksum. Algorithm borrowed from -log_block_calc_checksum. -@return checksum */ -UNIV_INLINE -ulint -log_online_calc_checksum( -/*=====================*/ - const byte* block) /*! 24) { - - sh = 0; - } - } - - return sum; -} - -/****************************************************************//** -Read one bitmap data page and check it for corruption. - -@return TRUE if page read OK, FALSE if I/O error */ -static -ibool -log_online_read_bitmap_page( -/*========================*/ - log_online_bitmap_file_t *bitmap_file, /*!size >= MODIFIED_PAGE_BLOCK_SIZE); - ut_a(bitmap_file->offset - <= bitmap_file->size - MODIFIED_PAGE_BLOCK_SIZE); - ut_a(bitmap_file->offset % MODIFIED_PAGE_BLOCK_SIZE == 0); - if (DB_SUCCESS != - os_file_read(IORequestRead, bitmap_file->file, page, - bitmap_file->offset, MODIFIED_PAGE_BLOCK_SIZE, - nullptr)) { - /* The following call prints an error message */ - os_file_get_last_error(TRUE); - msg("InnoDB: Warning: failed reading changed page bitmap " - "file \'%s\'", bitmap_file->name); - return FALSE; - } - - bitmap_file->offset += MODIFIED_PAGE_BLOCK_SIZE; - ut_ad(bitmap_file->offset <= bitmap_file->size); - - checksum = mach_read_from_4(page + MODIFIED_PAGE_BLOCK_CHECKSUM); - actual_checksum = log_online_calc_checksum(page); - *checksum_ok = (checksum == actual_checksum); - - return TRUE; -} - -/*********************************************************************//** -Check the name of a given file if it's a changed page bitmap file and -return file sequence and start LSN name components if it is. If is not, -the values of output parameters are undefined. - -@return TRUE if a given file is a changed page bitmap file. */ -static -ibool -log_online_is_bitmap_file( -/*======================*/ - const os_file_stat_t* file_info, /*!name) < OS_FILE_MAX_PATH); - - return ((file_info->type == OS_FILE_TYPE_FILE - || file_info->type == OS_FILE_TYPE_LINK) - && (sscanf(file_info->name, "%[a-z_]%lu_" LSN_PF ".xdb", stem, - bitmap_file_seq_num, bitmap_file_start_lsn) == 3) - && (!strcmp(stem, bmp_file_name_stem))); -} - -/*********************************************************************//** -List the bitmap files in srv_data_home and setup their range that contains the -specified LSN interval. This range, if non-empty, will start with a file that -has the greatest LSN equal to or less than the start LSN and will include all -the files up to the one with the greatest LSN less than the end LSN. Caller -must free bitmap_files->files when done if bitmap_files set to non-NULL and -this function returned TRUE. Field bitmap_files->count might be set to a -larger value than the actual count of the files, and space for the unused array -slots will be allocated but cleared to zeroes. - -@return TRUE if succeeded -*/ -static -ibool -log_online_setup_bitmap_file_range( -/*===============================*/ - log_online_bitmap_file_range_t *bitmap_files, /*!= range_start); - - bitmap_files->count = 0; - bitmap_files->files = NULL; - - /* 1st pass: size the info array */ - - bitmap_dir = os_file_opendir(srv_data_home); - if (UNIV_UNLIKELY(bitmap_dir == IF_WIN(INVALID_HANDLE_VALUE, NULL))) { - msg("InnoDB: Error: failed to open bitmap directory \'%s\'", - srv_data_home); - return FALSE; - } - - while (!os_file_readdir_next_file(srv_data_home, bitmap_dir, - &bitmap_dir_file_info)) { - - ulong file_seq_num; - lsn_t file_start_lsn; - - if (!log_online_is_bitmap_file(&bitmap_dir_file_info, - &file_seq_num, - &file_start_lsn) - || file_start_lsn >= range_end) { - - continue; - } - - if (file_seq_num > last_file_seq_num) { - - last_file_seq_num = file_seq_num; - } - - if (file_start_lsn >= range_start - || file_start_lsn == first_file_start_lsn - || first_file_start_lsn > range_start) { - - /* A file that falls into the range */ - - if (file_start_lsn < first_file_start_lsn) { - - first_file_start_lsn = file_start_lsn; - } - if (file_seq_num < first_file_seq_num) { - - first_file_seq_num = file_seq_num; - } - } else if (file_start_lsn > first_file_start_lsn) { - - /* A file that has LSN closer to the range start - but smaller than it, replacing another such file */ - first_file_start_lsn = file_start_lsn; - first_file_seq_num = file_seq_num; - } - } - - if (UNIV_UNLIKELY(os_file_closedir_failed(bitmap_dir))) { - os_file_get_last_error(TRUE); - msg("InnoDB: Error: cannot close \'%s\'",srv_data_home); - return FALSE; - } - - if (first_file_seq_num == ULONG_MAX && last_file_seq_num == 0) { - - bitmap_files->count = 0; - return TRUE; - } - - bitmap_files->count = last_file_seq_num - first_file_seq_num + 1; - - /* 2nd pass: get the file names in the file_seq_num order */ - - bitmap_dir = os_file_opendir(srv_data_home); - if (UNIV_UNLIKELY(bitmap_dir == IF_WIN(INVALID_HANDLE_VALUE, NULL))) { - msg("InnoDB: Error: failed to open bitmap directory \'%s\'", - srv_data_home); - return FALSE; - } - - bitmap_files->files = - static_cast - (malloc(bitmap_files->count * sizeof(bitmap_files->files[0]))); - memset(bitmap_files->files, 0, - bitmap_files->count * sizeof(bitmap_files->files[0])); - - while (!os_file_readdir_next_file(srv_data_home, bitmap_dir, - &bitmap_dir_file_info)) { - - ulong file_seq_num; - lsn_t file_start_lsn; - size_t array_pos; - - if (!log_online_is_bitmap_file(&bitmap_dir_file_info, - &file_seq_num, - &file_start_lsn) - || file_start_lsn >= range_end - || file_start_lsn < first_file_start_lsn) { - - continue; - } - - array_pos = file_seq_num - first_file_seq_num; - if (UNIV_UNLIKELY(array_pos >= bitmap_files->count)) { - - msg("InnoDB: Error: inconsistent bitmap file " - "directory"); - os_file_closedir(bitmap_dir); - free(bitmap_files->files); - return FALSE; - } - - if (file_seq_num > bitmap_files->files[array_pos].seq_num) { - - bitmap_files->files[array_pos].seq_num = file_seq_num; - strncpy(bitmap_files->files[array_pos].name, - bitmap_dir_file_info.name, FN_REFLEN - 1); - bitmap_files->files[array_pos].name[FN_REFLEN - 1] - = '\0'; - bitmap_files->files[array_pos].start_lsn - = file_start_lsn; - } - } - - if (UNIV_UNLIKELY(os_file_closedir_failed(bitmap_dir))) { - os_file_get_last_error(TRUE); - msg("InnoDB: Error: cannot close \'%s\'", srv_data_home); - free(bitmap_files->files); - return FALSE; - } - -#ifdef UNIV_DEBUG - ut_ad(bitmap_files->files[0].seq_num == first_file_seq_num); - - for (size_t i = 1; i < bitmap_files->count; i++) { - if (!bitmap_files->files[i].seq_num) { - - break; - } - ut_ad(bitmap_files->files[i].seq_num - > bitmap_files->files[i - 1].seq_num); - ut_ad(bitmap_files->files[i].start_lsn - >= bitmap_files->files[i - 1].start_lsn); - } -#endif - - return TRUE; -} - -/****************************************************************//** -Open a bitmap file for reading. - -@return whether opened successfully */ -static -bool -log_online_open_bitmap_file_read_only( -/*==================================*/ - const char* name, /*!name, FN_REFLEN, "%s%s", srv_data_home, name); - bitmap_file->file = os_file_create_simple_no_error_handling( - 0, bitmap_file->name, - OS_FILE_OPEN, OS_FILE_READ_ONLY, true, &success); - if (UNIV_UNLIKELY(!success)) { - - /* Here and below assume that bitmap file names do not - contain apostrophes, thus no need for ut_print_filename(). */ - msg("InnoDB: Warning: error opening the changed page " - "bitmap \'%s\'", bitmap_file->name); - return success; - } - - bitmap_file->size = os_file_get_size(bitmap_file->file); - bitmap_file->offset = 0; - -#ifdef __linux__ - posix_fadvise(bitmap_file->file, 0, 0, POSIX_FADV_SEQUENTIAL); - posix_fadvise(bitmap_file->file, 0, 0, POSIX_FADV_NOREUSE); -#endif - - return success; -} - -/****************************************************************//** -Diagnose one or both of the following situations if we read close to -the end of bitmap file: -1) Warn if the remainder of the file is less than one page. -2) Error if we cannot read any more full pages but the last read page -did not have the last-in-run flag set. - -@return FALSE for the error */ -static -ibool -log_online_diagnose_bitmap_eof( -/*===========================*/ - const log_online_bitmap_file_t* bitmap_file, /*!< in: bitmap file */ - ibool last_page_in_run)/*!< in: "last page in - run" flag value in the - last read page */ -{ - /* Check if we are too close to EOF to read a full page */ - if ((bitmap_file->size < MODIFIED_PAGE_BLOCK_SIZE) - || (bitmap_file->offset - > bitmap_file->size - MODIFIED_PAGE_BLOCK_SIZE)) { - - if (UNIV_UNLIKELY(bitmap_file->offset != bitmap_file->size)) { - - /* If we are not at EOF and we have less than one page - to read, it's junk. This error is not fatal in - itself. */ - - msg("InnoDB: Warning: junk at the end of changed " - "page bitmap file \'%s\'.", bitmap_file->name); - } - - if (UNIV_UNLIKELY(!last_page_in_run)) { - - /* We are at EOF but the last read page did not finish - a run */ - /* It's a "Warning" here because it's not a fatal error - for the whole server */ - msg("InnoDB: Warning: changed page bitmap " - "file \'%s\' does not contain a complete run " - "at the end.", bitmap_file->name); - return FALSE; - } - } - return TRUE; -} - -/* End of copy-pasted definitions */ - -/** Iterator structure over changed page bitmap */ -struct xb_page_bitmap_range_struct { - const xb_page_bitmap *bitmap; /* Bitmap with data */ - ulint space_id; /* Space id for this - iterator */ - ulint bit_i; /* Bit index of the iterator - position in the current page */ - const ib_rbt_node_t *bitmap_node; /* Current bitmap tree node */ - const byte *bitmap_page; /* Current bitmap page */ - ulint current_page_id;/* Current page id */ -}; - -/****************************************************************//** -Print a diagnostic message on missing bitmap data for an LSN range. */ -static -void -xb_msg_missing_lsn_data( -/*====================*/ - lsn_t missing_interval_start, /*!size >= MODIFIED_PAGE_BLOCK_SIZE); - - *page_end_lsn = 0; - - while ((*page_end_lsn <= lsn) - && (bitmap_file->offset - <= bitmap_file->size - MODIFIED_PAGE_BLOCK_SIZE)) { - - next_to_last_page_ok = last_page_ok; - if (!log_online_read_bitmap_page(bitmap_file, page, - &last_page_ok)) { - - return FALSE; - } - - *page_end_lsn = mach_read_from_8(page + MODIFIED_PAGE_END_LSN); - } - - /* We check two pages here because the last read page already contains - the required LSN data. If the next to the last one page is corrupted, - then we have no way of telling if that page contained the required LSN - range data too */ - return last_page_ok && next_to_last_page_ok; -} - -/****************************************************************//** -Read the disk bitmap and build the changed page bitmap tree for the -LSN interval incremental_lsn to log_sys.next_checkpoint_lsn. - -@return the built bitmap tree or NULL if unable to read the full interval for -any reason. */ -xb_page_bitmap* -xb_page_bitmap_init(void) -/*=====================*/ -{ - log_online_bitmap_file_t bitmap_file; - lsn_t bmp_start_lsn = incremental_lsn; - const lsn_t bmp_end_lsn{log_sys.next_checkpoint_lsn}; - byte page[MODIFIED_PAGE_BLOCK_SIZE]; - lsn_t current_page_end_lsn; - xb_page_bitmap *result; - ibool last_page_in_run= FALSE; - log_online_bitmap_file_range_t bitmap_files; - size_t bmp_i; - ibool last_page_ok = TRUE; - - if (UNIV_UNLIKELY(bmp_start_lsn > bmp_end_lsn)) { - - msg("mariabackup: incremental backup LSN " LSN_PF - " is larger than than the last checkpoint LSN " LSN_PF - , bmp_start_lsn, bmp_end_lsn); - return NULL; - } - - if (!log_online_setup_bitmap_file_range(&bitmap_files, bmp_start_lsn, - bmp_end_lsn)) { - - return NULL; - } - - /* Only accept no bitmap files returned if start LSN == end LSN */ - if (bitmap_files.count == 0 && bmp_end_lsn != bmp_start_lsn) { - - return NULL; - } - - result = rbt_create(MODIFIED_PAGE_BLOCK_SIZE, - log_online_compare_bmp_keys); - - if (bmp_start_lsn == bmp_end_lsn) { - - /* Empty range - empty bitmap */ - return result; - } - - bmp_i = 0; - - if (UNIV_UNLIKELY(bitmap_files.files[bmp_i].start_lsn - > bmp_start_lsn)) { - - /* The 1st file does not have the starting LSN data */ - xb_msg_missing_lsn_data(bmp_start_lsn, - bitmap_files.files[bmp_i].start_lsn); - rbt_free(result); - free(bitmap_files.files); - return NULL; - } - - /* Skip any zero-sized files at the start */ - while ((bmp_i < bitmap_files.count - 1) - && (bitmap_files.files[bmp_i].start_lsn - == bitmap_files.files[bmp_i + 1].start_lsn)) { - - bmp_i++; - } - - /* Is the 1st bitmap file missing? */ - if (UNIV_UNLIKELY(bitmap_files.files[bmp_i].name[0] == '\0')) { - - /* TODO: this is not the exact missing range */ - xb_msg_missing_lsn_data(bmp_start_lsn, bmp_end_lsn); - rbt_free(result); - free(bitmap_files.files); - return NULL; - } - - /* Open the 1st bitmap file */ - if (UNIV_UNLIKELY(!log_online_open_bitmap_file_read_only( - bitmap_files.files[bmp_i].name, - &bitmap_file))) { - - rbt_free(result); - free(bitmap_files.files); - return NULL; - } - - /* If the 1st file is truncated, no data. Not merged with the case - below because zero-length file indicates not a corruption but missing - subsequent files instead. */ - if (UNIV_UNLIKELY(bitmap_file.size < MODIFIED_PAGE_BLOCK_SIZE)) { - - xb_msg_missing_lsn_data(bmp_start_lsn, bmp_end_lsn); - rbt_free(result); - free(bitmap_files.files); - os_file_close(bitmap_file.file); - return NULL; - } - - /* Find the start of the required LSN range in the file */ - if (UNIV_UNLIKELY(!xb_find_lsn_in_bitmap_file(&bitmap_file, page, - ¤t_page_end_lsn, - bmp_start_lsn))) { - - msg("mariabackup: Warning: changed page bitmap file " - "\'%s\' corrupted", bitmap_file.name); - rbt_free(result); - free(bitmap_files.files); - os_file_close(bitmap_file.file); - return NULL; - } - - last_page_in_run - = mach_read_from_4(page + MODIFIED_PAGE_IS_LAST_BLOCK); - - if (UNIV_UNLIKELY(!log_online_diagnose_bitmap_eof(&bitmap_file, - last_page_in_run))) { - - rbt_free(result); - free(bitmap_files.files); - os_file_close(bitmap_file.file); - return NULL; - } - - if (UNIV_UNLIKELY(current_page_end_lsn < bmp_start_lsn)) { - - xb_msg_missing_lsn_data(current_page_end_lsn, bmp_start_lsn); - rbt_free(result); - free(bitmap_files.files); - os_file_close(bitmap_file.file); - return NULL; - } - - /* 1st bitmap page found, add it to the tree. */ - rbt_insert(result, page, page); - - /* Read next pages/files until all required data is read */ - while (last_page_ok - && (current_page_end_lsn < bmp_end_lsn - || (current_page_end_lsn == bmp_end_lsn - && !last_page_in_run))) { - - ib_rbt_bound_t tree_search_pos; - - /* If EOF, advance the file skipping over any empty files */ - while (bitmap_file.size < MODIFIED_PAGE_BLOCK_SIZE - || (bitmap_file.offset - > bitmap_file.size - MODIFIED_PAGE_BLOCK_SIZE)) { - - os_file_close(bitmap_file.file); - - if (UNIV_UNLIKELY( - !log_online_diagnose_bitmap_eof( - &bitmap_file, last_page_in_run))) { - - rbt_free(result); - free(bitmap_files.files); - return NULL; - } - - bmp_i++; - - if (UNIV_UNLIKELY(bmp_i == bitmap_files.count - || (bitmap_files.files[bmp_i].seq_num - == 0))) { - - xb_msg_missing_lsn_data(current_page_end_lsn, - bmp_end_lsn); - rbt_free(result); - free(bitmap_files.files); - return NULL; - } - - /* Is the next file missing? */ - if (UNIV_UNLIKELY(bitmap_files.files[bmp_i].name[0] - == '\0')) { - - /* TODO: this is not the exact missing range */ - xb_msg_missing_lsn_data(bitmap_files.files - [bmp_i - 1].start_lsn, - bmp_end_lsn); - rbt_free(result); - free(bitmap_files.files); - return NULL; - } - - if (UNIV_UNLIKELY( - !log_online_open_bitmap_file_read_only( - bitmap_files.files[bmp_i].name, - &bitmap_file))) { - - rbt_free(result); - free(bitmap_files.files); - return NULL; - } - } - - if (UNIV_UNLIKELY( - !log_online_read_bitmap_page(&bitmap_file, page, - &last_page_ok))) { - - rbt_free(result); - free(bitmap_files.files); - os_file_close(bitmap_file.file); - return NULL; - } - - if (UNIV_UNLIKELY(!last_page_ok)) { - - msg("mariabackup: warning: changed page bitmap file " - "\'%s\' corrupted.", bitmap_file.name); - rbt_free(result); - free(bitmap_files.files); - os_file_close(bitmap_file.file); - return NULL; - } - - /* Merge the current page with an existing page or insert a new - page into the tree */ - - if (!rbt_search(result, &tree_search_pos, page)) { - - /* Merge the bitmap pages */ - byte *existing_page - = rbt_value(byte, tree_search_pos.last); - bitmap_word_t *bmp_word_1 = (bitmap_word_t *) - (existing_page + MODIFIED_PAGE_BLOCK_BITMAP); - bitmap_word_t *bmp_end = (bitmap_word_t *) - (existing_page + MODIFIED_PAGE_BLOCK_UNUSED_2); - bitmap_word_t *bmp_word_2 = (bitmap_word_t *) - (page + MODIFIED_PAGE_BLOCK_BITMAP); - while (bmp_word_1 < bmp_end) { - - *bmp_word_1++ |= *bmp_word_2++; - } - xb_a (bmp_word_1 == bmp_end); - } else { - - /* Add a new page */ - rbt_add_node(result, &tree_search_pos, page); - } - - current_page_end_lsn - = mach_read_from_8(page + MODIFIED_PAGE_END_LSN); - last_page_in_run - = mach_read_from_4(page + MODIFIED_PAGE_IS_LAST_BLOCK); - } - - xb_a (current_page_end_lsn >= bmp_end_lsn); - - free(bitmap_files.files); - os_file_close(bitmap_file.file); - - return result; -} - -/****************************************************************//** -Free the bitmap tree. */ -void -xb_page_bitmap_deinit( -/*==================*/ - xb_page_bitmap* bitmap) /*!bitmap_page has been -already found/bumped by rbt_search()/rbt_next(). - -@return FALSE if no more bitmap data for the range space ID */ -static -ibool -xb_page_bitmap_setup_next_page( -/*===========================*/ - xb_page_bitmap_range* bitmap_range) /*!bitmap_node == NULL) { - - bitmap_range->current_page_id = ULINT_UNDEFINED; - return FALSE; - } - - bitmap_range->bitmap_page = rbt_value(byte, bitmap_range->bitmap_node); - - new_space_id = mach_read_from_4(bitmap_range->bitmap_page - + MODIFIED_PAGE_SPACE_ID); - if (new_space_id != bitmap_range->space_id) { - - /* No more data for the current page id. */ - xb_a(new_space_id > bitmap_range->space_id); - bitmap_range->current_page_id = ULINT_UNDEFINED; - return FALSE; - } - - new_1st_page_id = mach_read_from_4(bitmap_range->bitmap_page + - MODIFIED_PAGE_1ST_PAGE_ID); - xb_a (new_1st_page_id >= bitmap_range->current_page_id - || bitmap_range->current_page_id == ULINT_UNDEFINED); - - bitmap_range->current_page_id = new_1st_page_id; - bitmap_range->bit_i = 0; - - return TRUE; -} - -/** Find the node with the smallest key that greater than equal to search key. -@param[in] tree red-black tree -@param[in] key search key -@return node with the smallest greater-than-or-equal key -@retval NULL if none was found */ -static -const ib_rbt_node_t* -rbt_lower_bound(const ib_rbt_t* tree, const void* key) -{ - ut_ad(!tree->cmp_arg); - const ib_rbt_node_t* ge = NULL; - - for (const ib_rbt_node_t *node = tree->root->left; - node != tree->nil; ) { - int result = tree->compare(node->value, key); - - if (result < 0) { - node = node->right; - } else { - ge = node; - if (result == 0) { - break; - } - - node = node->left; - } - } - - return(ge); -} - -/****************************************************************//** -Set up a new bitmap range iterator over a given space id changed -pages in a given bitmap. - -@return bitmap range iterator */ -xb_page_bitmap_range* -xb_page_bitmap_range_init( -/*======================*/ - xb_page_bitmap* bitmap, /*!< in: bitmap to iterate over */ - ulint space_id) /*!< in: space id */ -{ - byte search_page[MODIFIED_PAGE_BLOCK_SIZE]; - xb_page_bitmap_range *result - = static_cast(malloc(sizeof(*result))); - - memset(result, 0, sizeof(*result)); - result->bitmap = bitmap; - result->space_id = space_id; - result->current_page_id = ULINT_UNDEFINED; - - /* Search for the 1st page for the given space id */ - /* This also sets MODIFIED_PAGE_1ST_PAGE_ID to 0, which is what we - want. */ - memset(search_page, 0, MODIFIED_PAGE_BLOCK_SIZE); - mach_write_to_4(search_page + MODIFIED_PAGE_SPACE_ID, space_id); - - result->bitmap_node = rbt_lower_bound(result->bitmap, search_page); - - xb_page_bitmap_setup_next_page(result); - - return result; -} - -/****************************************************************//** -Get the value of the bitmap->range->bit_i bitmap bit - -@return the current bit value */ -static inline -ibool -is_bit_set( -/*=======*/ - const xb_page_bitmap_range* bitmap_range) /*!< in: bitmap - range */ -{ - return ((*(((bitmap_word_t *)(bitmap_range->bitmap_page - + MODIFIED_PAGE_BLOCK_BITMAP)) - + (bitmap_range->bit_i >> 6))) - & (1ULL << (bitmap_range->bit_i & 0x3F))) ? TRUE : FALSE; -} - -/****************************************************************//** -Get the next page id that has its bit set or cleared, i.e. equal to -bit_value. - -@return page id */ -ulint -xb_page_bitmap_range_get_next_bit( -/*==============================*/ - xb_page_bitmap_range* bitmap_range, /*!< in/out: bitmap range */ - ibool bit_value) /*!< in: bit value */ -{ - if (UNIV_UNLIKELY(bitmap_range->current_page_id - == ULINT_UNDEFINED)) { - - return ULINT_UNDEFINED; - } - - do { - while (bitmap_range->bit_i < MODIFIED_PAGE_BLOCK_ID_COUNT) { - - while (is_bit_set(bitmap_range) != bit_value - && (bitmap_range->bit_i - < MODIFIED_PAGE_BLOCK_ID_COUNT)) { - - bitmap_range->current_page_id++; - bitmap_range->bit_i++; - } - - if (bitmap_range->bit_i - < MODIFIED_PAGE_BLOCK_ID_COUNT) { - - ulint result = bitmap_range->current_page_id; - bitmap_range->current_page_id++; - bitmap_range->bit_i++; - return result; - } - } - - bitmap_range->bitmap_node - = rbt_next(bitmap_range->bitmap, - bitmap_range->bitmap_node); - - } while (xb_page_bitmap_setup_next_page(bitmap_range)); - - return ULINT_UNDEFINED; -} - -/****************************************************************//** -Free the bitmap range iterator. */ -void -xb_page_bitmap_range_deinit( -/*========================*/ - xb_page_bitmap_range* bitmap_range) /*! in/out: bitmap range */ -{ - free(bitmap_range); -} diff --git a/extra/mariabackup/changed_page_bitmap.h b/extra/mariabackup/changed_page_bitmap.h deleted file mode 100644 index 8d504359..00000000 --- a/extra/mariabackup/changed_page_bitmap.h +++ /dev/null @@ -1,85 +0,0 @@ -/****************************************************** -XtraBackup: hot backup tool for InnoDB -(c) 2009-2012 Percona Inc. -Originally Created 3/3/2009 Yasufumi Kinoshita -Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, -Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA - -*******************************************************/ - -/* Changed page bitmap interface */ - -#ifndef XB_CHANGED_PAGE_BITMAP_H -#define XB_CHANGED_PAGE_BITMAP_H - -#include -#include - -/* The changed page bitmap structure */ -typedef ib_rbt_t xb_page_bitmap; - -struct xb_page_bitmap_range_struct; - -/* The bitmap range iterator over one space id */ -typedef struct xb_page_bitmap_range_struct xb_page_bitmap_range; - -/****************************************************************//** -Read the disk bitmap and build the changed page bitmap tree for the -LSN interval incremental_lsn to log_sys.next_checkpoint_lsn. - -@return the built bitmap tree */ -xb_page_bitmap* -xb_page_bitmap_init(void); -/*=====================*/ - -/****************************************************************//** -Free the bitmap tree. */ -void -xb_page_bitmap_deinit( -/*==================*/ - xb_page_bitmap* bitmap); /*! #include -#include #include #include @@ -143,7 +142,7 @@ static inline ATTRIBUTE_FORMAT(printf, 1,2) ATTRIBUTE_NORETURN void die(const ch # define POSIX_FADV_NORMAL # define POSIX_FADV_SEQUENTIAL # define POSIX_FADV_DONTNEED -# define posix_fadvise(a,b,c,d) do {} while(0) +# define posix_fadvise(fd, offset, len, advice) do { (void)offset; } while(0) #endif /*********************************************************************** diff --git a/extra/mariabackup/common_engine.cc b/extra/mariabackup/common_engine.cc new file mode 100644 index 00000000..a4a87062 --- /dev/null +++ b/extra/mariabackup/common_engine.cc @@ -0,0 +1,512 @@ +#include "common_engine.h" +#include "backup_copy.h" +#include "xtrabackup.h" +#include "common.h" +#include "backup_debug.h" + +#include +#include +#include +#include + +namespace common_engine { + +class Table { +public: + Table(std::string &db, std::string &table, std::string &fs_name) : + m_db(std::move(db)), m_table(std::move(table)), + m_fs_name(std::move(fs_name)) {} + virtual ~Table() {} + void add_file_name(const char *file_name) { m_fnames.push_back(file_name); } + virtual bool copy(ds_ctxt_t *ds, MYSQL *con, bool no_lock, + bool finalize, unsigned thread_num); + std::string &get_db() { return m_db; } + std::string &get_table() { return m_table; } + std::string &get_version() { return m_version; } + +protected: + std::string m_db; + std::string m_table; + std::string m_fs_name; + std::string m_version; + std::vector m_fnames; +}; + +bool +Table::copy(ds_ctxt_t *ds, MYSQL *con, bool no_lock, bool, unsigned thread_num) { + static const size_t buf_size = 10 * 1024 * 1024; + std::unique_ptr buf; + bool result = false; + File frm_file = -1; + std::vector files; + bool locked = false; + std::string full_tname("`"); + full_tname.append(m_db).append("`.`").append(m_table).append("`"); + + if (!no_lock && !backup_lock(con, full_tname.c_str())) { + msg(thread_num, "Error on executing BACKUP LOCK for table %s", + full_tname.c_str()); + goto exit; + } + else + locked = !no_lock; + + if ((frm_file = mysql_file_open(key_file_frm, (m_fs_name + ".frm").c_str(), + O_RDONLY | O_SHARE, MYF(0))) < 0 && !m_fnames.empty() && + !ends_with(m_fnames[0].c_str(), ".ARZ") && + !ends_with(m_fnames[0].c_str(), ".ARM")) { + // Don't treat it as error, as the table can be dropped after it + // was added to queue for copying + result = true; + goto exit; + } + + for (const auto &fname : m_fnames) { + File file = mysql_file_open(0, fname.c_str(),O_RDONLY | O_SHARE, MYF(0)); + if (file < 0) { + msg(thread_num, "Error on file %s open during %s table copy", + fname.c_str(), full_tname.c_str()); + goto exit; + } + files.push_back(file); + } + + if (locked && !backup_unlock(con)) { + msg(thread_num, "Error on BACKUP UNLOCK for table %s", full_tname.c_str()); + locked = false; + goto exit; + } + + locked = false; + + buf.reset(new uchar[buf_size]); + + for (size_t i = 0; i < m_fnames.size(); ++i) { + ds_file_t *dst_file = nullptr; + size_t bytes_read; + size_t copied_size = 0; + MY_STAT stat_info; + + if (my_fstat(files[i], &stat_info, MYF(0))) { + msg(thread_num, "error: failed to get stat info for file %s of " + "table %s", m_fnames[i].c_str(), full_tname.c_str()); + goto exit; + } + + const char *dst_path = + (xtrabackup_copy_back || xtrabackup_move_back) ? + m_fnames[i].c_str() : trim_dotslash(m_fnames[i].c_str()); + + dst_file = ds_open(ds, dst_path, &stat_info, false); + if (!dst_file) { + msg(thread_num, "error: cannot open destination stream for %s, table %s", + dst_path, full_tname.c_str()); + goto exit; + } + + while ((bytes_read = my_read(files[i], buf.get(), buf_size, MY_WME))) { + if (bytes_read == size_t(-1)) { + msg(thread_num, "error: file %s read for table %s", + m_fnames[i].c_str(), full_tname.c_str()); + ds_close(dst_file); + goto exit; + } + xtrabackup_io_throttling(); + if (ds_write(dst_file, buf.get(), bytes_read)) { + msg(thread_num, "error: file %s write for table %s", + dst_path, full_tname.c_str()); + ds_close(dst_file); + goto exit; + } + copied_size += bytes_read; + } + mysql_file_close(files[i], MYF(MY_WME)); + files[i] = -1; + ds_close(dst_file); + msg(thread_num, "Copied file %s for table %s, %zu bytes", + m_fnames[i].c_str(), full_tname.c_str(), copied_size); + } + + result = true; + +#ifndef DBUG_OFF + { + std::string sql_name(m_db); + sql_name.append("/").append(m_table); + DBUG_MARIABACKUP_EVENT_LOCK("after_ce_table_copy", fil_space_t::name_type(sql_name.data(), sql_name.size())); + } +#endif // DBUG_OFF +exit: + if (frm_file >= 0) { + m_version = ::read_table_version_id(frm_file); + mysql_file_close(frm_file, MYF(MY_WME)); + } + if (locked && !backup_unlock(con)) { + msg(thread_num, "Error on BACKUP UNLOCK for table %s", full_tname.c_str()); + result = false; + } + for (auto file : files) + if (file >= 0) + mysql_file_close(file, MYF(MY_WME)); + return result; +} + +// Append-only tables +class LogTable : public Table { + public: + LogTable(std::string &db, std::string &table, std::string &fs_name) : + Table(db, table, fs_name) {} + + virtual ~LogTable() { (void)close(); } + bool + copy(ds_ctxt_t *ds, MYSQL *con, bool no_lock, bool finalize, + unsigned thread_num) override; + bool close(); + private: + bool open(ds_ctxt_t *ds, unsigned thread_num); + std::vector m_src; + std::vector m_dst; +}; + +bool +LogTable::open(ds_ctxt_t *ds, unsigned thread_num) { + DBUG_ASSERT(m_src.empty()); + DBUG_ASSERT(m_dst.empty()); + + std::string full_tname("`"); + full_tname.append(m_db).append("`.`").append(m_table).append("`"); + + for (const auto &fname : m_fnames) { + File file = mysql_file_open(0, fname.c_str(),O_RDONLY | O_SHARE, MYF(0)); + if (file < 0) { + msg(thread_num, "Error on file %s open during %s log table copy", + fname.c_str(), full_tname.c_str()); + return false; + } + m_src.push_back(file); + + MY_STAT stat_info; + if (my_fstat(file, &stat_info, MYF(0))) { + msg(thread_num, "error: failed to get stat info for file %s of " + "log table %s", fname.c_str(), full_tname.c_str()); + return false; + } + const char *dst_path = + (xtrabackup_copy_back || xtrabackup_move_back) ? + fname.c_str() : trim_dotslash(fname.c_str()); + ds_file_t *dst_file = ds_open(ds, dst_path, &stat_info, false); + if (!dst_file) { + msg(thread_num, "error: cannot open destination stream for %s, " + "log table %s", dst_path, full_tname.c_str()); + return false; + } + m_dst.push_back(dst_file); + } + + File frm_file; + if ((frm_file = mysql_file_open(key_file_frm, (m_fs_name + ".frm").c_str(), + O_RDONLY | O_SHARE, MYF(0))) < 0 && !m_fnames.empty() && + !ends_with(m_fnames[0].c_str(), ".ARZ") && + !ends_with(m_fnames[0].c_str(), ".ARM")) { + msg(thread_num, "Error on .frm file open for log table %s", + full_tname.c_str()); + return false; + } + + m_version = ::read_table_version_id(frm_file); + mysql_file_close(frm_file, MYF(MY_WME)); + + return true; +} + +bool LogTable::close() { + while (!m_src.empty()) { + auto f = m_src.back(); + m_src.pop_back(); + mysql_file_close(f, MYF(MY_WME)); + } + while (!m_dst.empty()) { + auto f = m_dst.back(); + m_dst.pop_back(); + ds_close(f); + } + return true; +} + +bool +LogTable::copy(ds_ctxt_t *ds, MYSQL *con, bool no_lock, bool finalize, + unsigned thread_num) { + static const size_t buf_size = 10 * 1024 * 1024; + DBUG_ASSERT(ds); + DBUG_ASSERT(con); + if (m_src.empty() && !open(ds, thread_num)) { + close(); + return false; + } + DBUG_ASSERT(m_src.size() == m_dst.size()); + + std::unique_ptr buf(new uchar[buf_size]); + for (size_t i = 0; i < m_src.size(); ++i) { + // .CSM can be rewritten (see write_meta_file() usage in ha_tina.cc) + if (!finalize && ends_with(m_fnames[i].c_str(), ".CSM")) + continue; + size_t bytes_read; + size_t copied_size = 0; + while ((bytes_read = my_read(m_src[i], buf.get(), buf_size, MY_WME))) { + if (bytes_read == size_t(-1)) { + msg(thread_num, "error: file %s read for log table %s", + m_fnames[i].c_str(), + std::string("`").append(m_db).append("`.`"). + append(m_table).append("`").c_str()); + close(); + return false; + } + xtrabackup_io_throttling(); + if (ds_write(m_dst[i], buf.get(), bytes_read)) { + msg(thread_num, "error: file %s write for log table %s", + m_fnames[i].c_str(), std::string("`").append(m_db).append("`.`"). + append(m_table).append("`").c_str()); + close(); + return false; + } + copied_size += bytes_read; + } + msg(thread_num, "Copied file %s for log table %s, %zu bytes", + m_fnames[i].c_str(), std::string("`").append(m_db).append("`.`"). + append(m_table).append("`").c_str(), copied_size); + } + + return true; +} + +class BackupImpl { + public: + BackupImpl( + const char *datadir_path, ds_ctxt_t *datasink, + std::vector &con_pool, ThreadPool &thread_pool) : + m_datadir_path(datadir_path), m_ds(datasink), m_con_pool(con_pool), + m_process_table_jobs(thread_pool) {} + ~BackupImpl() { } + bool scan( + const std::unordered_set &exclude_tables, + std::unordered_set *out_processed_tables, + bool no_lock, bool collect_log_and_stats); + void set_post_copy_table_hook(const post_copy_table_hook_t &hook) { + m_table_post_copy_hook = hook; + } + bool copy_log_tables(bool finalize); + bool copy_stats_tables(); + bool wait_for_finish(); + bool close_log_tables(); + private: + + void process_table_job(Table *table, bool no_lock, bool delete_table, + bool finalize, unsigned thread_num); + + const char *m_datadir_path; + ds_ctxt_t *m_ds; + std::vector &m_con_pool; + TasksGroup m_process_table_jobs; + + post_copy_table_hook_t m_table_post_copy_hook; + std::unordered_map> m_log_tables; + std::unordered_map> m_stats_tables; +}; + +void BackupImpl::process_table_job(Table *table, bool no_lock, + bool delete_table, bool finalize, unsigned thread_num) { + int result = 0; + + if (!m_process_table_jobs.get_result()) + goto exit; + + if (!table->copy(m_ds, m_con_pool[thread_num], no_lock, finalize, thread_num)) + goto exit; + + if (m_table_post_copy_hook) + m_table_post_copy_hook(table->get_db(), table->get_table(), + table->get_version()); + + result = 1; + +exit: + if (delete_table) + delete table; + m_process_table_jobs.finish_task(result); +} + +bool BackupImpl::scan(const std::unordered_set &exclude_tables, + std::unordered_set *out_processed_tables, bool no_lock, + bool collect_log_and_stats) { + + msg("Start scanning common engine tables, need backup locks: %d, " + "collect log and stat tables: %d", no_lock, collect_log_and_stats); + + std::unordered_map> found_tables; + + foreach_file_in_db_dirs(m_datadir_path, + [&](const char *file_path)->bool { + + static const char *ext_list[] = + {".MYD", ".MYI", ".MRG", ".ARM", ".ARZ", ".CSM", ".CSV", NULL}; + + bool is_aria = ends_with(file_path, ".MAD") || ends_with(file_path, ".MAI"); + + if (!collect_log_and_stats && is_aria) + return true; + + if (!is_aria && !filename_matches(file_path, ext_list)) + return true; + + if (check_if_skip_table(file_path)) { + msg("Skipping %s.", file_path); + return true; + } + + auto db_table_fs = convert_filepath_to_tablename(file_path); + auto tk = + table_key(std::get<0>(db_table_fs), std::get<1>(db_table_fs)); + + // log and stats tables are only collected in this function, + // so there is no need to filter out them with exclude_tables. + if (collect_log_and_stats) { + if (is_log_table(std::get<0>(db_table_fs).c_str(), + std::get<1>(db_table_fs).c_str())) { + auto table_it = m_log_tables.find(tk); + if (table_it == m_log_tables.end()) { + msg("Log table found: %s", tk.c_str()); + table_it = m_log_tables.emplace(tk, + std::unique_ptr(new LogTable(std::get<0>(db_table_fs), + std::get<1>(db_table_fs), std::get<2>(db_table_fs)))).first; + } + msg("Collect log table file: %s", file_path); + table_it->second->add_file_name(file_path); + return true; + } + // Aria can handle statistics tables + else if (is_stats_table(std::get<0>(db_table_fs).c_str(), + std::get<1>(db_table_fs).c_str()) && !is_aria) { + auto table_it = m_stats_tables.find(tk); + if (table_it == m_stats_tables.end()) { + msg("Stats table found: %s", tk.c_str()); + table_it = m_stats_tables.emplace(tk, + std::unique_ptr
(new Table(std::get<0>(db_table_fs), + std::get<1>(db_table_fs), std::get<2>(db_table_fs)))).first; + } + msg("Collect stats table file: %s", file_path); + table_it->second->add_file_name(file_path); + return true; + } + } else if (is_log_table(std::get<0>(db_table_fs).c_str(), + std::get<1>(db_table_fs).c_str()) || + is_stats_table(std::get<0>(db_table_fs).c_str(), + std::get<1>(db_table_fs).c_str())) + return true; + + if (is_aria) + return true; + + if (exclude_tables.count(tk)) { + msg("Skip table %s at it is in exclude list", tk.c_str()); + return true; + } + + auto table_it = found_tables.find(tk); + if (table_it == found_tables.end()) { + table_it = found_tables.emplace(tk, + std::unique_ptr
(new Table(std::get<0>(db_table_fs), + std::get<1>(db_table_fs), std::get<2>(db_table_fs)))).first; + } + + table_it->second->add_file_name(file_path); + + return true; + }); + + for (auto &table_it : found_tables) { + m_process_table_jobs.push_task( + std::bind(&BackupImpl::process_table_job, this, table_it.second.release(), + no_lock, true, false, std::placeholders::_1)); + if (out_processed_tables) + out_processed_tables->insert(table_it.first); + } + + msg("Stop scanning common engine tables"); + return true; +} + +bool BackupImpl::copy_log_tables(bool finalize) { + for (auto &table_it : m_log_tables) { + // Do not execute BACKUP LOCK for log tables as it's supposed + // that they must be copied on BLOCK_DDL and BLOCK_COMMIT locks. + m_process_table_jobs.push_task( + std::bind(&BackupImpl::process_table_job, this, table_it.second.get(), + true, false, finalize, std::placeholders::_1)); + } + return true; +} + +bool BackupImpl::copy_stats_tables() { + for (auto &table_it : m_stats_tables) { + // Do not execute BACKUP LOCK for stats tables as it's supposed + // that they must be copied on BLOCK_DDL and BLOCK_COMMIT locks. + // Delete stats table object after copy (see process_table_job()) + m_process_table_jobs.push_task( + std::bind(&BackupImpl::process_table_job, this, table_it.second.release(), + true, true, false, std::placeholders::_1)); + } + m_stats_tables.clear(); + return true; +} + +bool BackupImpl::wait_for_finish() { + /* Wait for threads to exit */ + return m_process_table_jobs.wait_for_finish(); +} + +bool BackupImpl::close_log_tables() { + bool result = wait_for_finish(); + for (auto &table_it : m_log_tables) + table_it.second->close(); + return result; +} + +Backup::Backup(const char *datadir_path, ds_ctxt_t *datasink, + std::vector &con_pool, ThreadPool &thread_pool) : + m_backup_impl( + new BackupImpl(datadir_path, datasink, con_pool, + thread_pool)) { } + +Backup::~Backup() { + delete m_backup_impl; +} + +bool Backup::scan( + const std::unordered_set &exclude_tables, + std::unordered_set *out_processed_tables, + bool no_lock, bool collect_log_and_stats) { + return m_backup_impl->scan(exclude_tables, out_processed_tables, no_lock, + collect_log_and_stats); +} + +bool Backup::copy_log_tables(bool finalize) { + return m_backup_impl->copy_log_tables(finalize); +} + +bool Backup::copy_stats_tables() { + return m_backup_impl->copy_stats_tables(); +} + +bool Backup::wait_for_finish() { + return m_backup_impl->wait_for_finish(); +} + +bool Backup::close_log_tables() { + return m_backup_impl->close_log_tables(); +} + +void Backup::set_post_copy_table_hook(const post_copy_table_hook_t &hook) { + m_backup_impl->set_post_copy_table_hook(hook); +} + +} // namespace common_engine diff --git a/extra/mariabackup/common_engine.h b/extra/mariabackup/common_engine.h new file mode 100644 index 00000000..6f5d8062 --- /dev/null +++ b/extra/mariabackup/common_engine.h @@ -0,0 +1,39 @@ +#pragma once +#include "my_global.h" +#include "backup_mysql.h" +#include "datasink.h" +#include "thread_pool.h" +#include "xtrabackup.h" + +#include +#include +#include + +namespace common_engine { + +class BackupImpl; + +class Backup { + public: + Backup(const char *datadir_path, ds_ctxt_t *datasink, + std::vector &con_pool, ThreadPool &thread_pool); + ~Backup(); + Backup (Backup &&other) = delete; + Backup & operator= (Backup &&other) = delete; + Backup(const Backup &) = delete; + Backup & operator= (const Backup &) = delete; + bool scan( + const std::unordered_set &exclude_tables, + std::unordered_set *out_processed_tables, + bool no_lock, bool collect_log_and_stats); + bool copy_log_tables(bool finalize); + bool copy_stats_tables(); + bool wait_for_finish(); + bool close_log_tables(); + void set_post_copy_table_hook(const post_copy_table_hook_t &hook); + private: + BackupImpl *m_backup_impl; +}; + +} // namespace common_engine + diff --git a/extra/mariabackup/datasink.cc b/extra/mariabackup/datasink.cc index a576526d..132ff3fc 100644 --- a/extra/mariabackup/datasink.cc +++ b/extra/mariabackup/datasink.cc @@ -80,11 +80,11 @@ ds_create(const char *root, ds_type_t type) /************************************************************************ Open a datasink file */ ds_file_t * -ds_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat) +ds_open(ds_ctxt_t *ctxt, const char *path, const MY_STAT *stat, bool rewrite) { ds_file_t *file; - file = ctxt->datasink->open(ctxt, path, stat); + file = ctxt->datasink->open(ctxt, path, stat, rewrite); if (file != NULL) { file->datasink = ctxt->datasink; } @@ -104,6 +104,30 @@ ds_write(ds_file_t *file, const void *buf, size_t len) return file->datasink->write(file, (const uchar *)buf, len); } +int ds_seek_set(ds_file_t *file, my_off_t offset) { + DBUG_ASSERT(file); + DBUG_ASSERT(file->datasink); + if (file->datasink->seek_set) + return file->datasink->seek_set(file, offset); + return 0; +} + +int ds_rename(ds_ctxt_t *ctxt, const char *old_path, const char *new_path) { + DBUG_ASSERT(ctxt); + DBUG_ASSERT(ctxt->datasink); + if (ctxt->datasink->rename) + return ctxt->datasink->rename(ctxt, old_path, new_path); + return 0; +} + +int ds_remove(ds_ctxt_t *ctxt, const char *path) { + DBUG_ASSERT(ctxt); + DBUG_ASSERT(ctxt->datasink); + if (ctxt->datasink->remove) + return ctxt->datasink->mremove(ctxt, path); + return 0; +} + /************************************************************************ Close a datasink file. @return 0 on success, 1, on error. */ diff --git a/extra/mariabackup/datasink.h b/extra/mariabackup/datasink.h index 57468e0c..98cbe525 100644 --- a/extra/mariabackup/datasink.h +++ b/extra/mariabackup/datasink.h @@ -43,7 +43,8 @@ typedef struct ds_ctxt { */ bool copy_file(const char *src_file_path, const char *dst_file_path, - uint thread_n); + uint thread_n, + bool rewrite = false); bool move_file(const char *src_file_path, const char *dst_file_path, @@ -76,10 +77,15 @@ typedef struct { struct datasink_struct { ds_ctxt_t *(*init)(const char *root); - ds_file_t *(*open)(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat); + ds_file_t *(*open)(ds_ctxt_t *ctxt, const char *path, + const MY_STAT *stat, bool rewrite); int (*write)(ds_file_t *file, const unsigned char *buf, size_t len); + int (*seek_set)(ds_file_t *file, my_off_t offset); int (*close)(ds_file_t *file); int (*remove)(const char *path); + // TODO: consider to return bool from "rename" and "remove" + int (*rename)(ds_ctxt_t *ctxt, const char *old_path, const char *new_path); + int (*mremove)(ds_ctxt_t *ctxt, const char *path); void (*deinit)(ds_ctxt_t *ctxt); }; @@ -106,12 +112,17 @@ ds_ctxt_t *ds_create(const char *root, ds_type_t type); /************************************************************************ Open a datasink file */ -ds_file_t *ds_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat); +ds_file_t *ds_open( + ds_ctxt_t *ctxt, const char *path, const MY_STAT *stat, bool rewrite = false); /************************************************************************ Write to a datasink file. @return 0 on success, 1 on error. */ int ds_write(ds_file_t *file, const void *buf, size_t len); +int ds_seek_set(ds_file_t *file, my_off_t offset); + +int ds_rename(ds_ctxt_t *ctxt, const char *old_path, const char *new_path); +int ds_remove(ds_ctxt_t *ctxt, const char *path); /************************************************************************ Close a datasink file. diff --git a/extra/mariabackup/ddl_log.cc b/extra/mariabackup/ddl_log.cc new file mode 100644 index 00000000..6af34172 --- /dev/null +++ b/extra/mariabackup/ddl_log.cc @@ -0,0 +1,553 @@ +#include "ddl_log.h" +#include "common.h" +#include "my_sys.h" +#include "sql_table.h" +#include "backup_copy.h" +#include "xtrabackup.h" +#include +#include +#include +#include + +namespace ddl_log { + +struct Entry { + enum Type { + CREATE, + ALTER, + RENAME, + REPAIR, + OPTIMIZE, + DROP, + TRUNCATE, + CHANGE_INDEX, + BULK_INSERT + }; + Type type; + std::string date; + std::string engine; + bool partitioned; + std::string db; + std::string table; + std::string id; + std::string new_engine; + bool new_partitioned; + std::string new_db; + std::string new_table; + std::string new_id; +}; + +typedef std::vector> entries_t; +typedef std::function)> store_entry_func_t; + +const char *aria_engine_name = "Aria"; +static const char *frm_ext = ".frm"; +static const char *database_keyword = "DATABASE"; + +const std::unordered_map> engine_exts = +{ + {"Aria", {".MAD", ".MAI"}}, + {"MyISAM", {".MYD", ".MYI"}}, + {"MRG_MyISAM", {".MRG"}}, + {"ARCHIVE", {".ARM", ".ARZ"}}, + {"CSV", {".CSM", ".CSV"}} +}; + +static inline bool known_engine(const std::string &engine) { + return engine_exts.count(engine); +} + +// TODO: add error messages +size_t parse(const uchar *buf, size_t buf_size, bool &error_flag, + store_entry_func_t &store_entry_func) { + DBUG_ASSERT(buf); + static constexpr char token_delimiter = '\t'; + static constexpr char line_delimiter = '\n'; + enum { + TOKEN_FIRST = 0, + TOKEN_DATE = TOKEN_FIRST, + TOKEN_TYPE, + TOKEN_ENGINE, + TOKEN_PARTITIONED, + TOKEN_DB, + TOKEN_TABLE, + TOKEN_ID, + TOKEN_MANDATORY = TOKEN_ID, + TOKEN_NEW_ENGINE, + TOKEN_NEW_PARTITIONED, + TOKEN_NEW_DB, + TOKEN_NEW_TABLE, + TOKEN_NEW_ID, + TOKEN_LAST = TOKEN_NEW_ID + }; + const size_t string_offsets[TOKEN_LAST + 1] = { + offsetof(Entry, date), + offsetof(Entry, type), // not a string, be careful + offsetof(Entry, engine), + offsetof(Entry, partitioned), // not a string, be careful + offsetof(Entry, db), + offsetof(Entry, table), + offsetof(Entry, id), + offsetof(Entry, new_engine), + offsetof(Entry, new_partitioned), // not a string, be careful + offsetof(Entry, new_db), + offsetof(Entry, new_table), + offsetof(Entry, new_id) + }; + const std::unordered_map str_to_type = { + {"CREATE", Entry::CREATE}, + {"ALTER", Entry::ALTER}, + {"RENAME", Entry::RENAME}, + // TODO: fix to use uppercase-only + {"repair", Entry::REPAIR}, + {"optimize", Entry::OPTIMIZE}, + {"DROP", Entry::DROP}, + {"TRUNCATE", Entry::TRUNCATE}, + {"CHANGE_INDEX", Entry::CHANGE_INDEX}, + {"BULK_INSERT", Entry::BULK_INSERT} + }; + + const uchar *new_line = buf; + const uchar *token_start = buf; + unsigned token_num = TOKEN_FIRST; + + error_flag = false; + + std::unique_ptr entry(new Entry()); + + for (const uchar *ptr = buf; ptr < buf + buf_size; ++ptr) { + + if (*ptr != token_delimiter && *ptr != line_delimiter) + continue; + + if (token_start != ptr) { + std::string token(token_start, ptr); + + if (token_num == TOKEN_TYPE) { + const auto type_it = str_to_type.find(token); + if (type_it == str_to_type.end()) { + error_flag = true; + goto exit; + } + entry->type = type_it->second; + } + else if (token_num == TOKEN_PARTITIONED) { + entry->partitioned = token[0] - '0'; + } + else if (token_num == TOKEN_NEW_PARTITIONED) { + entry->new_partitioned = token[0] - '0'; + } + else if (token_num <= TOKEN_LAST) { + DBUG_ASSERT(token_num != TOKEN_TYPE); + DBUG_ASSERT(token_num != TOKEN_PARTITIONED); + DBUG_ASSERT(token_num != TOKEN_NEW_PARTITIONED); + reinterpret_cast + (reinterpret_cast(entry.get()) + string_offsets[token_num])-> + assign(std::move(token)); + } + else { + error_flag = true; + goto exit; + } + } + token_start = ptr + 1; + + if (*ptr == line_delimiter) { + if (token_num < TOKEN_MANDATORY) { + error_flag = true; + goto exit; + } + if (!store_entry_func(std::move(entry))) { + error_flag = true; + goto exit; + } + entry.reset(new Entry()); + token_num = TOKEN_FIRST; + new_line = ptr + 1; + } else + ++token_num; + } + +exit: + return new_line - buf; +} + +bool parse(const char *file_path, store_entry_func_t store_entry_func) { + DBUG_ASSERT(file_path); + DBUG_ASSERT(store_entry_func); + File file= -1; + bool result = true; + uchar buf[1024]; + size_t bytes_read = 0; + size_t buf_read_offset = 0; + + if ((file= my_open(file_path, O_RDONLY | O_SHARE | O_NOFOLLOW | O_CLOEXEC, + MYF(MY_WME))) < 0) { + msg("DDL log file %s open failed: %d", file_path, my_errno); + result = false; + goto exit; + } + + while((bytes_read = my_read( + file, &buf[buf_read_offset], sizeof(buf) - buf_read_offset, MY_WME)) > 0) { + if (bytes_read == size_t(-1)) { + msg("DDL log file %s read error: %d", file_path, my_errno); + result = false; + break; + } + bytes_read += buf_read_offset; + bool parse_error_flag = false; + size_t bytes_parsed = parse( + buf, bytes_read, parse_error_flag, store_entry_func); + if (parse_error_flag) { + result = false; + break; + } + size_t rest_size = bytes_read - bytes_parsed; + if (rest_size) + memcpy(buf, buf + bytes_parsed, rest_size); + buf_read_offset = rest_size; + } + +exit: + if (file >= 0) + my_close(file, MYF(MY_WME)); + return result; +}; + + +static +bool process_database( + const char *datadir_path, + ds_ctxt_t *ds, + const Entry &entry, + std::unordered_set &dropped_databases) { + + if (entry.type == Entry::Type::CREATE || + entry.type == Entry::Type::ALTER) { + std::string opt_file(datadir_path); + opt_file.append("/").append(entry.db).append("/db.opt"); + if (!ds->copy_file(opt_file.c_str(), opt_file.c_str(), 0, true)) { + msg("Failed to re-copy %s.", opt_file.c_str()); + return false; + } + if (entry.type == Entry::Type::CREATE) + dropped_databases.erase(entry.db); + return true; + } + + DBUG_ASSERT(entry.type == Entry::Type::DROP); + + std::string db_path(datadir_path); + db_path.append("/").append(entry.db); + const char *dst_path = convert_dst(db_path.c_str()); + if (!ds_remove(ds, dst_path)) { + dropped_databases.insert(entry.db); + return true; + } + return false; +} + +static +std::unique_ptr> + find_table_files( + const char *dir_path, + const std::string &db, + const std::string &table) { + + std::unique_ptr> + result(new std::vector()); + + std::string prefix = convert_tablename_to_filepath(dir_path, db, table); + foreach_file_in_db_dirs(dir_path, [&](const char *file_name)->bool { + if (!strncmp(file_name, prefix.c_str(), prefix.size())) { + DBUG_ASSERT(strlen(file_name) >= prefix.size()); + if (file_name[prefix.size()] == '.' || + !strncmp(file_name + prefix.size(), "#P#", strlen("#P#"))) + result->push_back(std::string(file_name)); + } + return true; + }); + + return result; +} + +static +bool process_remove( + const char *datadir_path, + ds_ctxt_t *ds, + const Entry &entry, + bool remove_frm) { + + if (check_if_skip_table( + std::string(entry.db).append("/").append(entry.table).c_str())) + return true; + + auto ext_it = engine_exts.find(entry.engine); + if (ext_it == engine_exts.end()) + return true; + + std::string file_preffix = convert_tablename_to_filepath(datadir_path, + entry.db, entry.table); + const char *dst_preffix = convert_dst(file_preffix.c_str()); + + for (const char *ext : ext_it->second) { + std::string old_name(dst_preffix); + if (!entry.partitioned) + old_name.append(ext); + else + old_name.append("#P#*"); + if (ds_remove(ds, old_name.c_str())) { + msg("Failed to remove %s.", old_name.c_str()); + return false; + } + } + + if (remove_frm) { + std::string old_frm_name(dst_preffix); + old_frm_name.append(frm_ext); + if (ds_remove(ds, old_frm_name.c_str())) { + msg("Failed to remove %s.", old_frm_name.c_str()); + return false; + } + } + return true; + +} + +static +bool process_recopy( + const char *datadir_path, + ds_ctxt_t *ds, + const Entry &entry, + const tables_t &tables) { + + if (check_if_skip_table( + std::string(entry.db).append("/").append(entry.table).c_str())) + return true; + + const std::string &new_table_id = + entry.new_id.empty() ? entry.id : entry.new_id; + DBUG_ASSERT(!new_table_id.empty()); + const std::string &new_table = + entry.new_table.empty() ? entry.table : entry.new_table; + DBUG_ASSERT(!new_table.empty()); + const std::string &new_db = + entry.new_db.empty() ? entry.db : entry.new_db; + DBUG_ASSERT(!new_db.empty()); + const std::string &new_engine = + entry.new_engine.empty() ? entry.engine : entry.new_engine; + DBUG_ASSERT(!new_engine.empty()); + + if (entry.type != Entry::Type::BULK_INSERT) { + auto table_it = tables.find(table_key(new_db, new_table)); + if (table_it != tables.end() && + table_it->second == new_table_id) + return true; + } + + if (!entry.new_engine.empty() && + entry.engine != entry.new_engine && + !known_engine(entry.new_engine)) { + return process_remove(datadir_path, ds, entry, false); + } + + if ((entry.partitioned || entry.new_partitioned) && + !process_remove(datadir_path, ds, entry, false)) + return false; + + if (entry.partitioned || entry.new_partitioned) { + auto files = find_table_files(datadir_path, new_db, new_table); + if (!files.get()) + return true; + for (const auto &file : *files) { + const char *dst_path = convert_dst(file.c_str()); + if (!ds->copy_file(file.c_str(), dst_path, 0, true)) { + msg("Failed to re-copy %s.", file.c_str()); + return false; + } + } + return true; + } + + auto ext_it = engine_exts.find(new_engine); + if (ext_it == engine_exts.end()) + return false; + + for (const char *ext : ext_it->second) { + std::string file_name = + convert_tablename_to_filepath(datadir_path, new_db, new_table). + append(ext); + const char *dst_path = convert_dst(file_name.c_str()); + if (file_exists(file_name.c_str()) && + !ds->copy_file(file_name.c_str(), dst_path, 0, true)) { + msg("Failed to re-copy %s.", file_name.c_str()); + return false; + } + } + + std::string frm_file = + convert_tablename_to_filepath(datadir_path, new_db, new_table). + append(frm_ext); + const char *frm_dst_path = convert_dst(frm_file.c_str()); + if (file_exists(frm_file.c_str()) && + !ds->copy_file(frm_file.c_str(), frm_dst_path, 0, true)) { + msg("Failed to re-copy %s.", frm_file.c_str()); + return false; + } + + return true; +} + +static +bool process_rename( + const char *datadir_path, + ds_ctxt_t *ds, + const Entry &entry) { + + if (check_if_skip_table( + std::string(entry.db).append("/").append(entry.table).c_str())) + return true; + + DBUG_ASSERT(entry.db != "partition"); + + auto ext_it = engine_exts.find(entry.engine); + if (ext_it == engine_exts.end()) + return false; + + std::string new_preffix = convert_tablename_to_filepath(datadir_path, + entry.new_db, entry.new_table); + const char *dst_path = convert_dst(new_preffix.c_str()); + + std::string old_preffix = convert_tablename_to_filepath(datadir_path, + entry.db, entry.table); + const char *src_path = convert_dst(old_preffix.c_str()); + + for (const char *ext : ext_it->second) { + std::string old_name(src_path); + old_name.append(ext); + std::string new_name(dst_path); + new_name.append(ext); + if (ds_rename(ds, old_name.c_str(), new_name.c_str())) { + msg("Failed to rename %s to %s.", + old_name.c_str(), new_name.c_str()); + return false; + } + } + + std::string new_frm_file = new_preffix + frm_ext; + const char *new_frm_dst = convert_dst(new_frm_file.c_str()); + if (file_exists(new_frm_file.c_str()) && + !ds->copy_file(new_frm_file.c_str(), new_frm_dst, 0, true)) { + msg("Failed to re-copy %s.", new_frm_file.c_str()); + return false; + } + +// TODO: return this code if .frm is copied not under BLOCK_DDL +/* + std::string old_frm_name(src_path); + old_frm_name.append(frm_ext); + std::string new_frm_name(dst_path); + new_frm_name.append(frm_ext); + if (ds_rename(ds, old_frm_name.c_str(), new_frm_name.c_str())) { + msg("Failed to rename %s to %s.", + old_frm_name.c_str(), new_frm_name.c_str()); + return false; + } +*/ + return true; +} + +bool backup( + const char *datadir_path, + ds_ctxt_t *ds, + const tables_t &tables) { + DBUG_ASSERT(datadir_path); + DBUG_ASSERT(ds); + char ddl_log_path[FN_REFLEN]; + fn_format(ddl_log_path, "ddl", datadir_path, ".log", 0); + std::vector> entries; + + std::unordered_set processed_tables; + std::unordered_set dropped_databases; + + bool parsing_result = + parse(ddl_log_path, [&](std::unique_ptr entry)->bool { + + if (entry->engine == database_keyword) + return process_database(datadir_path, ds, *entry, dropped_databases); + + if (!known_engine(entry->engine) && !known_engine(entry->new_engine)) + return true; + + if (entry->type == Entry::Type::CREATE || + (entry->type == Entry::Type::ALTER && + !entry->new_engine.empty() && + entry->engine != entry->new_engine)) { + if (!process_recopy(datadir_path, ds, *entry, tables)) + return false; + processed_tables.insert(table_key(entry->db, entry->table)); + if (entry->type == Entry::Type::ALTER) + processed_tables.insert(table_key(entry->new_db, entry->new_table)); + return true; + } + + if (entry->type == Entry::Type::DROP) { + if (!process_remove(datadir_path, ds, *entry, true)) + return false; + processed_tables.insert(table_key(entry->db, entry->table)); + return true; + } + if (entry->type == Entry::Type::RENAME) { + if (entry->partitioned) { + if (!process_remove(datadir_path, ds, *entry, true)) + return false; + Entry recopy_entry { + entry->type, + {}, + entry->new_engine.empty() ? entry->engine : entry->new_engine, + true, + entry->new_db, + entry->new_table, + entry->new_id, + {}, true, {}, {}, {} + }; + if (!process_recopy(datadir_path, ds, recopy_entry, tables)) + return false; + } + else if (!process_rename(datadir_path, ds, *entry)) + return false; + processed_tables.insert(table_key(entry->db, entry->table)); + processed_tables.insert(table_key(entry->new_db, entry->new_table)); + return true; + } + + entries.push_back(std::move(entry)); + return true; + + }); + + if (!parsing_result) + return false; + + + while (!entries.empty()) { + auto entry = std::move(entries.back()); + entries.pop_back(); + auto tk = table_key( + entry->new_db.empty() ? entry->db : entry->new_db, + entry->new_table.empty() ? entry->table : entry->new_table); + if (dropped_databases.count(entry->db) || + dropped_databases.count(entry->new_db)) + continue; + if (processed_tables.count(tk)) + continue; + processed_tables.insert(std::move(tk)); + if (!process_recopy(datadir_path, ds, *entry, tables)) + return false; + } + + return true; +} + +} // namespace ddl_log diff --git a/extra/mariabackup/ddl_log.h b/extra/mariabackup/ddl_log.h new file mode 100644 index 00000000..5cac3e5d --- /dev/null +++ b/extra/mariabackup/ddl_log.h @@ -0,0 +1,15 @@ +#pragma once +#include "my_global.h" +#include "datasink.h" +#include "aria_backup_client.h" +#include +#include +#include +#include + +namespace ddl_log { + +typedef std::unordered_map tables_t; +bool backup(const char *datadir_path, ds_ctxt_t *ds, const tables_t &tables); + +} // namespace ddl_log diff --git a/extra/mariabackup/ds_buffer.cc b/extra/mariabackup/ds_buffer.cc index d6a42095..bc1d4663 100644 --- a/extra/mariabackup/ds_buffer.cc +++ b/extra/mariabackup/ds_buffer.cc @@ -44,7 +44,7 @@ typedef struct { static ds_ctxt_t *buffer_init(const char *root); static ds_file_t *buffer_open(ds_ctxt_t *ctxt, const char *path, - MY_STAT *mystat); + const MY_STAT *mystat, bool rewrite); static int buffer_write(ds_file_t *file, const uchar *buf, size_t len); static int buffer_close(ds_file_t *file); static void buffer_deinit(ds_ctxt_t *ctxt); @@ -53,8 +53,11 @@ datasink_t datasink_buffer = { &buffer_init, &buffer_open, &buffer_write, + nullptr, &buffer_close, &dummy_remove, + nullptr, + nullptr, &buffer_deinit }; @@ -84,8 +87,10 @@ buffer_init(const char *root) } static ds_file_t * -buffer_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) +buffer_open(ds_ctxt_t *ctxt, const char *path, + const MY_STAT *mystat, bool rewrite) { + DBUG_ASSERT(rewrite == false); ds_buffer_ctxt_t *buffer_ctxt; ds_ctxt_t *pipe_ctxt; ds_file_t *dst_file; diff --git a/extra/mariabackup/ds_compress.cc b/extra/mariabackup/ds_compress.cc index f7a9b7a1..0cb52e97 100644 --- a/extra/mariabackup/ds_compress.cc +++ b/extra/mariabackup/ds_compress.cc @@ -65,7 +65,7 @@ extern ulonglong xtrabackup_compress_chunk_size; static ds_ctxt_t *compress_init(const char *root); static ds_file_t *compress_open(ds_ctxt_t *ctxt, const char *path, - MY_STAT *mystat); + const MY_STAT *mystat, bool rewrite); static int compress_write(ds_file_t *file, const uchar *buf, size_t len); static int compress_close(ds_file_t *file); static void compress_deinit(ds_ctxt_t *ctxt); @@ -74,8 +74,11 @@ datasink_t datasink_compress = { &compress_init, &compress_open, &compress_write, + nullptr, &compress_close, &dummy_remove, + nullptr, + nullptr, &compress_deinit }; @@ -116,8 +119,10 @@ compress_init(const char *root) static ds_file_t * -compress_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) +compress_open(ds_ctxt_t *ctxt, const char *path, + const MY_STAT *mystat, bool rewrite) { + DBUG_ASSERT(rewrite == false); ds_compress_ctxt_t *comp_ctxt; ds_ctxt_t *dest_ctxt; ds_file_t *dest_file; diff --git a/extra/mariabackup/ds_local.cc b/extra/mariabackup/ds_local.cc index f86612b9..ff2021fc 100644 --- a/extra/mariabackup/ds_local.cc +++ b/extra/mariabackup/ds_local.cc @@ -42,8 +42,9 @@ typedef struct { static ds_ctxt_t *local_init(const char *root); static ds_file_t *local_open(ds_ctxt_t *ctxt, const char *path, - MY_STAT *mystat); + const MY_STAT *mystat, bool rewrite); static int local_write(ds_file_t *file, const uchar *buf, size_t len); +static int local_seek_set(ds_file_t *file, my_off_t offset); static int local_close(ds_file_t *file); static void local_deinit(ds_ctxt_t *ctxt); @@ -52,13 +53,20 @@ static int local_remove(const char *path) return unlink(path); } +static int local_rename( + ds_ctxt_t *ctxt, const char *old_path, const char *new_path); +static int local_mremove(ds_ctxt_t *ctxt, const char *path); + extern "C" { datasink_t datasink_local = { &local_init, &local_open, &local_write, + &local_seek_set, &local_close, &local_remove, + &local_rename, + &local_mremove, &local_deinit }; } @@ -89,7 +97,7 @@ local_init(const char *root) static ds_file_t * local_open(ds_ctxt_t *ctxt, const char *path, - MY_STAT *mystat __attribute__((unused))) + const MY_STAT *mystat __attribute__((unused)), bool rewrite) { char fullpath[FN_REFLEN]; char dirpath[FN_REFLEN]; @@ -111,8 +119,10 @@ local_open(ds_ctxt_t *ctxt, const char *path, return NULL; } - fd = my_create(fullpath, 0, O_WRONLY | O_BINARY | O_EXCL | O_NOFOLLOW, - MYF(MY_WME)); + // TODO: check in Windows and set the corresponding flags on fail + fd = my_create(fullpath, 0, + O_WRONLY | O_BINARY | (rewrite ? O_TRUNC : O_EXCL) | O_NOFOLLOW, + MYF(MY_WME)); if (fd < 0) { return NULL; } @@ -194,8 +204,8 @@ static void init_ibd_data(ds_local_file_t *local_file, const uchar *buf, size_t return; } - auto flags = mach_read_from_4(&buf[FIL_PAGE_DATA + FSP_SPACE_FLAGS]); - auto ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); + uint32_t flags = mach_read_from_4(&buf[FIL_PAGE_DATA + FSP_SPACE_FLAGS]); + uint32_t ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); local_file->pagesize= ssize == 0 ? UNIV_PAGE_SIZE_ORIG : ((UNIV_ZIP_SIZE_MIN >> 1) << ssize); local_file->compressed = fil_space_t::full_crc32(flags) ? fil_space_t::is_compressed(flags) @@ -239,6 +249,15 @@ local_write(ds_file_t *file, const uchar *buf, size_t len) return 1; } +static +int +local_seek_set(ds_file_t *file, my_off_t offset) { + ds_local_file_t *local_file= (ds_local_file_t *)file->ptr; + if (my_seek(local_file->fd, offset, SEEK_SET, MYF(0)) == MY_FILEPOS_ERROR) + return 1; + return 0; +} + /* Set EOF at file's current position.*/ static int set_eof(File fd) { @@ -276,3 +295,77 @@ local_deinit(ds_ctxt_t *ctxt) my_free(ctxt->root); my_free(ctxt); } + + +static int local_rename( + ds_ctxt_t *ctxt, const char *old_path, const char *new_path) { + char full_old_path[FN_REFLEN]; + char full_new_path[FN_REFLEN]; + fn_format(full_old_path, old_path, ctxt->root, "", MYF(MY_RELATIVE_PATH)); + fn_format(full_new_path, new_path, ctxt->root, "", MYF(MY_RELATIVE_PATH)); + // Ignore errors as .frm files can me copied separately. + // TODO: return error processing here after the corresponding changes in + // xtrabackup.cc + (void)my_rename(full_old_path, full_new_path, MYF(0)); +// if (my_rename(full_old_path, full_new_path, MYF(0))) { +// msg("Failed to rename file %s to %s", old_path, new_path); +// return 1; +// } + return 0; +} + +// It's ok if destination does not contain the file or folder +static int local_mremove(ds_ctxt_t *ctxt, const char *path) { + char full_path[FN_REFLEN]; + fn_format(full_path, path, ctxt->root, "", MYF(MY_RELATIVE_PATH)); + size_t full_path_len = strlen(full_path); + if (full_path[full_path_len - 1] == '*') { + full_path[full_path_len - 1] = '\0'; + char *preffix = strrchr(full_path, '/'); + const char *full_path_dir = full_path; + size_t preffix_len; + if (preffix) { + preffix_len = (full_path_len - 1) - (preffix - full_path); + *(preffix++) = '\0'; + } + else { + preffix = full_path; + preffix_len = full_path_len - 1; + full_path_dir= IF_WIN(".\\", "./"); + } + if (!preffix_len) + return 0; + MY_DIR *dir= my_dir(full_path_dir, 0); + if (!dir) + return 0; + for (size_t i = 0; i < dir->number_of_files; ++i) { + char full_fpath[FN_REFLEN]; + if (strncmp(dir->dir_entry[i].name, preffix, preffix_len)) + continue; + fn_format(full_fpath, dir->dir_entry[i].name, + full_path_dir, "", MYF(MY_RELATIVE_PATH)); + (void)my_delete(full_fpath, MYF(0)); + } + my_dirend(dir); + } + else { + MY_STAT stat; + if (!my_stat(full_path, &stat, MYF(0))) + return 0; + MY_DIR *dir= my_dir(full_path, 0); + if (!dir) { + // TODO: check for error here if necessary + (void)my_delete(full_path, MYF(0)); + return 0; + } + for (size_t i = 0; i < dir->number_of_files; ++i) { + char full_fpath[FN_REFLEN]; + fn_format(full_fpath, dir->dir_entry[i].name, + full_path, "", MYF(MY_RELATIVE_PATH)); + (void)my_delete(full_fpath, MYF(0)); + } + my_dirend(dir); + (void)my_rmtree(full_path, MYF(0)); + } + return 0; +} diff --git a/extra/mariabackup/ds_stdout.cc b/extra/mariabackup/ds_stdout.cc index a9639ff7..3fc0873b 100644 --- a/extra/mariabackup/ds_stdout.cc +++ b/extra/mariabackup/ds_stdout.cc @@ -30,7 +30,7 @@ typedef struct { static ds_ctxt_t *stdout_init(const char *root); static ds_file_t *stdout_open(ds_ctxt_t *ctxt, const char *path, - MY_STAT *mystat); + const MY_STAT *mystat, bool rewrite); static int stdout_write(ds_file_t *file, const uchar *buf, size_t len); static int stdout_close(ds_file_t *file); static void stdout_deinit(ds_ctxt_t *ctxt); @@ -39,8 +39,11 @@ datasink_t datasink_stdout = { &stdout_init, &stdout_open, &stdout_write, + nullptr, &stdout_close, &dummy_remove, + nullptr, + nullptr, &stdout_deinit }; @@ -61,8 +64,9 @@ static ds_file_t * stdout_open(ds_ctxt_t *ctxt __attribute__((unused)), const char *path __attribute__((unused)), - MY_STAT *mystat __attribute__((unused))) + const MY_STAT *mystat __attribute__((unused)), bool rewrite) { + DBUG_ASSERT(rewrite == false); ds_stdout_file_t *stdout_file; ds_file_t *file; size_t pathlen; diff --git a/extra/mariabackup/ds_tmpfile.cc b/extra/mariabackup/ds_tmpfile.cc index 80b9d3bb..6bafee25 100644 --- a/extra/mariabackup/ds_tmpfile.cc +++ b/extra/mariabackup/ds_tmpfile.cc @@ -41,7 +41,7 @@ typedef struct { static ds_ctxt_t *tmpfile_init(const char *root); static ds_file_t *tmpfile_open(ds_ctxt_t *ctxt, const char *path, - MY_STAT *mystat); + const MY_STAT *mystat, bool rewrite); static int tmpfile_write(ds_file_t *file, const uchar *buf, size_t len); static int tmpfile_close(ds_file_t *file); static void tmpfile_deinit(ds_ctxt_t *ctxt); @@ -50,8 +50,11 @@ datasink_t datasink_tmpfile = { &tmpfile_init, &tmpfile_open, &tmpfile_write, + nullptr, &tmpfile_close, &dummy_remove, + nullptr, + nullptr, &tmpfile_deinit }; @@ -80,8 +83,9 @@ tmpfile_init(const char *root) static ds_file_t * tmpfile_open(ds_ctxt_t *ctxt, const char *path, - MY_STAT *mystat) + const MY_STAT *mystat, bool rewrite) { + DBUG_ASSERT(rewrite == false); ds_tmpfile_ctxt_t *tmpfile_ctxt; char tmp_path[FN_REFLEN]; ds_tmp_file_t *tmp_file; diff --git a/extra/mariabackup/ds_xbstream.cc b/extra/mariabackup/ds_xbstream.cc index 3bf8bd08..96e0cf7a 100644 --- a/extra/mariabackup/ds_xbstream.cc +++ b/extra/mariabackup/ds_xbstream.cc @@ -40,24 +40,31 @@ General streaming interface */ static ds_ctxt_t *xbstream_init(const char *root); static ds_file_t *xbstream_open(ds_ctxt_t *ctxt, const char *path, - MY_STAT *mystat); + const MY_STAT *mystat, bool rewrite); static int xbstream_write(ds_file_t *file, const uchar *buf, size_t len); +static int xbstream_seek_set(ds_file_t *file, my_off_t offset); static int xbstream_close(ds_file_t *file); static void xbstream_deinit(ds_ctxt_t *ctxt); +static int xbstream_rename( + ds_ctxt_t *ctxt, const char *old_path, const char *new_path); +static int xbstream_mremove(ds_ctxt_t *ctxt, const char *path); + datasink_t datasink_xbstream = { &xbstream_init, &xbstream_open, &xbstream_write, + &xbstream_seek_set, &xbstream_close, &dummy_remove, + &xbstream_rename, + &xbstream_mremove, &xbstream_deinit }; static ssize_t -my_xbstream_write_callback(xb_wstream_file_t *f __attribute__((unused)), - void *userdata, const void *buf, size_t len) +my_xbstream_write_callback(void *userdata, const void *buf, size_t len) { ds_stream_ctxt_t *stream_ctxt; @@ -89,7 +96,7 @@ xbstream_init(const char *root __attribute__((unused))) goto err; } - xbstream = xb_stream_write_new(); + xbstream = xb_stream_write_new(my_xbstream_write_callback, stream_ctxt); if (xbstream == NULL) { msg("xb_stream_write_new() failed."); goto err; @@ -108,7 +115,8 @@ err: static ds_file_t * -xbstream_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) +xbstream_open(ds_ctxt_t *ctxt, const char *path, + const MY_STAT *mystat, bool rewrite) { ds_file_t *file; ds_stream_file_t *stream_file; @@ -144,9 +152,7 @@ xbstream_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) xbstream = stream_ctxt->xbstream; - xbstream_file = xb_stream_write_open(xbstream, path, mystat, - stream_ctxt, - my_xbstream_write_callback); + xbstream_file = xb_stream_write_open(xbstream, path, mystat, rewrite); if (xbstream_file == NULL) { msg("xb_stream_write_open() failed."); @@ -190,6 +196,45 @@ xbstream_write(ds_file_t *file, const uchar *buf, size_t len) return 0; } +static +int +xbstream_seek_set(ds_file_t *file, my_off_t offset) +{ + ds_stream_file_t *stream_file; + xb_wstream_file_t *xbstream_file; + + + stream_file = (ds_stream_file_t *) file->ptr; + + xbstream_file = stream_file->xbstream_file; + + if (xb_stream_write_seek_set(xbstream_file, offset)) { + msg("xb_stream_write_seek_set() failed."); + return 1; + } + + return 0; +} + +static +int +xbstream_mremove(ds_ctxt_t *ctxt, const char *path) { + ds_stream_ctxt_t *stream_ctxt = + reinterpret_cast(ctxt->ptr); + xb_wstream_t *xbstream = stream_ctxt->xbstream; + return xb_stream_write_remove(xbstream, path); +} + +static +int +xbstream_rename( + ds_ctxt_t *ctxt, const char *old_path, const char *new_path) { + ds_stream_ctxt_t *stream_ctxt = + reinterpret_cast(ctxt->ptr); + xb_wstream_t *xbstream = stream_ctxt->xbstream; + return xb_stream_write_rename(xbstream, old_path, new_path); +} + static int xbstream_close(ds_file_t *file) diff --git a/extra/mariabackup/encryption_plugin.cc b/extra/mariabackup/encryption_plugin.cc new file mode 100644 index 00000000..ab0c5140 --- /dev/null +++ b/extra/mariabackup/encryption_plugin.cc @@ -0,0 +1,249 @@ +/* Copyright (c) 2017, MariaDB Corporation. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +extern struct st_maria_plugin *mysql_optional_plugins[]; +extern struct st_maria_plugin *mysql_mandatory_plugins[]; +static void encryption_plugin_init(int argc, char **argv); + +extern char *xb_plugin_load; +extern char *xb_plugin_dir; + +const int PLUGIN_MAX_ARGS = 1024; +std::vector backup_plugins_args; + +const char *QUERY_PLUGIN = +"SELECT plugin_name, plugin_library, @@plugin_dir" +" FROM information_schema.plugins WHERE plugin_type='ENCRYPTION'" +" OR (plugin_type = 'DAEMON' AND plugin_name LIKE 'provider\\_%')" +" AND plugin_status='ACTIVE'"; + +std::string encryption_plugin_config; + +static void add_to_plugin_load_list(const char *plugin_def) +{ + opt_plugin_load_list_ptr->push_back(new i_string(plugin_def)); +} + +static char XTRABACKUP_EXE[] = "xtrabackup"; + +/* + Read "plugin-load" value (encryption plugin) from backup-my.cnf during + prepare phase. + The value is stored during backup phase. +*/ +static std::string get_encryption_plugin_from_cnf() +{ + FILE *f = fopen("backup-my.cnf", "r"); + if (!f) + { + die("Can't open backup-my.cnf for reading"); + } + char line[512]; + std::string plugin_load; + while (fgets(line, sizeof(line), f)) + { + if (strncmp(line, "plugin_load=", 12) == 0) + { + plugin_load = line + 12; + // remote \n at the end of string + plugin_load.resize(plugin_load.size() - 1); + } + + if (strncmp(line, "innodb_encrypt_tables=", 22) == 0) + { + if (!strncmp(line + 22, "ON", 2) || + !strncmp(line + 22, "1", 1)) + srv_encrypt_tables= 1; + else if (!strncmp(line + 22, "FORCE", 5) || + !strncmp(line + 22, "2", 1)) + srv_encrypt_tables= 2; + } + } + fclose(f); + return plugin_load; +} + + +void encryption_plugin_backup_init(MYSQL *mysql) +{ + MYSQL_RES *result; + MYSQL_ROW row; + std::ostringstream oss; + char *argv[PLUGIN_MAX_ARGS]; + char show_query[1024] = ""; + std::string plugin_load; + int argc; + + result = xb_mysql_query(mysql, QUERY_PLUGIN, true, true); + while ((row = mysql_fetch_row(result))) + { + char *name= row[0]; + char *library= row[1]; + char *dir= row[2]; + + if (!plugin_load.length()) + { +#ifdef _WIN32 + for (char *p = dir; *p; p++) + if (*p == '\\') *p = '/'; +#endif + strncpy(opt_plugin_dir, dir, FN_REFLEN - 1); + opt_plugin_dir[FN_REFLEN - 1] = '\0'; + oss << "plugin_dir=" << '"' << dir << '"' << std::endl; + } + + plugin_load += std::string(";") + name; + + if (library) + { + /* Remove shared library suffixes, in case we'll prepare on different OS.*/ + const char *extensions[] = { ".dll", ".so", 0 }; + for (size_t i = 0; extensions[i]; i++) + { + const char *ext = extensions[i]; + if (ends_with(library, ext)) + library[strlen(library) - strlen(ext)] = 0; + } + plugin_load += std::string("=") + library; + } + + if (strncmp(name, "provider_", 9) == 0) + continue; + + /* Read plugin variables. */ + snprintf(show_query, sizeof(show_query), "SHOW variables like '%s_%%'", name); + } + mysql_free_result(result); + if (!plugin_load.length()) + return; + + oss << "plugin_load=" << plugin_load.c_str() + 1 << std::endl; + + /* Required to load the plugin later.*/ + add_to_plugin_load_list(plugin_load.c_str() + 1); + + + if (*show_query) + { + result = xb_mysql_query(mysql, show_query, true, true); + while ((row = mysql_fetch_row(result))) + { + std::string arg("--"); + arg += row[0]; + arg += "="; + arg += row[1]; + backup_plugins_args.push_back(arg); + oss << row[0] << "=" << row[1] << std::endl; + } + + mysql_free_result(result); + + /* Check whether to encrypt logs. */ + result = xb_mysql_query(mysql, "select @@innodb_encrypt_log", true, true); + row = mysql_fetch_row(result); + srv_encrypt_log = (row != 0 && row[0][0] == '1'); + oss << "innodb_encrypt_log=" << row[0] << std::endl; + + mysql_free_result(result); + } + + result = xb_mysql_query(mysql, "select @@innodb_encrypt_tables", true, true); + row = mysql_fetch_row(result); + if (!row); + else if (const char *r= row[0]) + { + if (!strcmp(r, "ON")) srv_encrypt_tables= 1; + else if (!strcmp(r, "FORCE")) srv_encrypt_tables= 2; + oss << "innodb_encrypt_tables=" << r << std::endl; + } + + mysql_free_result(result); + encryption_plugin_config = oss.str(); + + argc = 0; + argv[argc++] = XTRABACKUP_EXE; + for(size_t i = 0; i < backup_plugins_args.size(); i++) + { + argv[argc++] = (char *)backup_plugins_args[i].c_str(); + if (argc == PLUGIN_MAX_ARGS - 2) + break; + } + argv[argc] = 0; + + encryption_plugin_init(argc, argv); +} + +const char *encryption_plugin_get_config() +{ + return encryption_plugin_config.c_str(); +} + +extern int finalize_encryption_plugin(st_plugin_int *plugin); + + +void encryption_plugin_prepare_init(int argc, char **argv) +{ + std::string plugin_load= get_encryption_plugin_from_cnf(); + if (plugin_load.size()) + { + msg("Loading encryption plugin from %s", plugin_load.c_str()); + } + else + { + finalize_encryption_plugin(0); + return; + } + + add_to_plugin_load_list(plugin_load.c_str()); + + if (xb_plugin_dir) + { + strncpy(opt_plugin_dir, xb_plugin_dir, FN_REFLEN - 1); + opt_plugin_dir[FN_REFLEN - 1] = '\0'; + } + + char **new_argv = new char *[argc + 2]; + new_argv[0] = XTRABACKUP_EXE; + memcpy(&new_argv[1], argv, argc*sizeof(char *)); + + encryption_plugin_init(argc+1, new_argv); + + delete[] new_argv; +} + +static void encryption_plugin_init(int argc, char **argv) +{ + /* Patch optional and mandatory plugins, we only need to load the one in xb_plugin_load. */ + mysql_optional_plugins[0] = mysql_mandatory_plugins[0] = 0; + plugin_maturity = MariaDB_PLUGIN_MATURITY_UNKNOWN; /* mariabackup accepts all plugins */ + msg("Loading encryption plugin"); + for (int i= 1; i < argc; i++) + msg("\t Encryption plugin parameter : '%s'", argv[i]); + plugin_init(&argc, argv, PLUGIN_INIT_SKIP_PLUGIN_TABLE); +} + diff --git a/extra/mariabackup/encryption_plugin.h b/extra/mariabackup/encryption_plugin.h new file mode 100644 index 00000000..16d74790 --- /dev/null +++ b/extra/mariabackup/encryption_plugin.h @@ -0,0 +1,7 @@ +#include +#include +extern void encryption_plugin_backup_init(MYSQL *mysql); +extern const char* encryption_plugin_get_config(); +extern void encryption_plugin_prepare_init(int argc, char **argv); + +//extern void encryption_plugin_init(int argc, char **argv); diff --git a/extra/mariabackup/fil_cur.cc b/extra/mariabackup/fil_cur.cc index 4f5d67a5..be871e4a 100644 --- a/extra/mariabackup/fil_cur.cc +++ b/extra/mariabackup/fil_cur.cc @@ -199,14 +199,6 @@ xb_fil_cur_open( return(XB_FIL_CUR_SKIP); } -#ifdef HAVE_FCNTL_DIRECT - if (srv_file_flush_method == SRV_O_DIRECT - || srv_file_flush_method == SRV_O_DIRECT_NO_FSYNC) { - - os_file_set_nocache(cursor->file, node->name, "OPEN"); - } -#endif - posix_fadvise(cursor->file, 0, 0, POSIX_FADV_SEQUENTIAL); cursor->page_size = node->space->physical_size(); @@ -239,12 +231,14 @@ xb_fil_cur_open( / cursor->page_size); cursor->read_filter = read_filter; - cursor->read_filter->init(&cursor->read_filter_ctxt, cursor, - node->space->id); + cursor->read_filter->init(&cursor->read_filter_ctxt, cursor); return(XB_FIL_CUR_SUCCESS); } +/* Stack usage 131224 with clang */ +PRAGMA_DISABLE_CHECK_STACK_FRAME + static bool page_is_corrupted(const byte *page, ulint page_no, const xb_fil_cur_t *cursor, const fil_space_t *space) @@ -348,6 +342,7 @@ static bool page_is_corrupted(const byte *page, ulint page_no, return buf_page_is_corrupted(true, page, space->flags); } +PRAGMA_REENABLE_CHECK_STACK_FRAME /** Reads and verifies the next block of pages from the source file. Positions the cursor after the last read non-corrupted page. @@ -510,10 +505,6 @@ xb_fil_cur_close( /*=============*/ xb_fil_cur_t *cursor) /*!< in/out: source file cursor */ { - if (cursor->read_filter) { - cursor->read_filter->deinit(&cursor->read_filter_ctxt); - } - aligned_free(cursor->buf); cursor->buf = NULL; diff --git a/extra/mariabackup/fil_cur.h b/extra/mariabackup/fil_cur.h index b7812f65..46c8cb03 100644 --- a/extra/mariabackup/fil_cur.h +++ b/extra/mariabackup/fil_cur.h @@ -27,6 +27,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA #include #include "read_filt.h" +#include "mtr0types.h" #include "srv0start.h" #include "srv0srv.h" #include "xtrabackup.h" diff --git a/extra/mariabackup/innobackupex.cc b/extra/mariabackup/innobackupex.cc index b925b415..2de57a14 100644 --- a/extra/mariabackup/innobackupex.cc +++ b/extra/mariabackup/innobackupex.cc @@ -78,10 +78,8 @@ my_bool opt_ibx_galera_info = FALSE; my_bool opt_ibx_slave_info = FALSE; my_bool opt_ibx_no_lock = FALSE; my_bool opt_ibx_safe_slave_backup = FALSE; -my_bool opt_ibx_rsync = FALSE; my_bool opt_ibx_force_non_empty_dirs = FALSE; my_bool opt_ibx_noversioncheck = FALSE; -my_bool opt_ibx_no_backup_locks = FALSE; my_bool opt_ibx_decompress = FALSE; char *opt_ibx_incremental_history_name = NULL; @@ -268,8 +266,10 @@ static struct my_option ibx_long_options[] = (uchar *) &opt_ibx_incremental, (uchar *) &opt_ibx_incremental, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"no-lock", OPT_NO_LOCK, "Use this option to disable table lock " - "with \"FLUSH TABLES WITH READ LOCK\". Use it only if ALL your " + {"no-lock", OPT_NO_LOCK, "This option should not be used as " + "mariadb-backup now is using BACKUP LOCKS, which minimizes the " + "lock time. ALTER TABLE can run in parallel with BACKUP LOCKS." + "Use the --no-lock option it only if ALL your " "tables are InnoDB and you DO NOT CARE about the binary log " "position of the backup. This option shouldn't be used if there " "are any DDL statements being executed or if any updates are " @@ -297,15 +297,6 @@ static struct my_option ibx_long_options[] = (uchar *) &opt_ibx_safe_slave_backup, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"rsync", OPT_RSYNC, "Uses the rsync utility to optimize local file " - "transfers. When this option is specified, innobackupex uses rsync " - "to copy all non-InnoDB files instead of spawning a separate cp for " - "each file, which can be much faster for servers with a large number " - "of databases or tables. This option cannot be used together with " - "--stream.", - (uchar *) &opt_ibx_rsync, (uchar *) &opt_ibx_rsync, - 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"force-non-empty-directories", OPT_FORCE_NON_EMPTY_DIRS, "This " "option, when specified, makes --copy-back or --move-back transfer " "files to non-empty directories. Note that no existing files will be " @@ -330,13 +321,9 @@ static struct my_option ibx_long_options[] = (uchar *) &opt_ibx_noversioncheck, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"no-backup-locks", OPT_NO_BACKUP_LOCKS, "This option controls if " - "backup locks should be used instead of FLUSH TABLES WITH READ LOCK " - "on the backup stage. The option has no effect when backup locks are " - "not supported by the server. This option is enabled by default, " - "disable with --no-backup-locks.", - (uchar *) &opt_ibx_no_backup_locks, - (uchar *) &opt_ibx_no_backup_locks, + {"no-backup-locks", OPT_NO_BACKUP_LOCKS, + "Old disabled option which has no effect anymore.", + (uchar *) 0, (uchar*) 0, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"decompress", OPT_DECOMPRESS, "Decompresses all files with the .qp " @@ -402,11 +389,10 @@ static struct my_option ibx_long_options[] = REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"ftwrl-wait-query-type", OPT_LOCK_WAIT_QUERY_TYPE, - "This option specifies which types of queries are allowed to complete " - "before innobackupex will issue the global lock. Default is all.", - (uchar*) &opt_ibx_lock_wait_query_type, - (uchar*) &opt_ibx_lock_wait_query_type, &query_type_typelib, - GET_ENUM, REQUIRED_ARG, QUERY_TYPE_ALL, 0, 0, 0, 0, 0}, + "Old disabled option which has no effect anymore (not needed " + "with BACKUP LOCKS)", + (uchar*) 0, (uchar*) 0, &query_type_typelib, GET_ENUM, + REQUIRED_ARG, QUERY_TYPE_ALL, 0, 0, 0, 0, 0}, {"kill-long-query-type", OPT_KILL_LONG_QUERY_TYPE, "This option specifies which types of queries should be killed to " @@ -447,32 +433,32 @@ static struct my_option ibx_long_options[] = REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"kill-long-queries-timeout", OPT_KILL_LONG_QUERIES_TIMEOUT, - "This option specifies the number of seconds innobackupex waits " - "between starting FLUSH TABLES WITH READ LOCK and killing those " - "queries that block it. Default is 0 seconds, which means " - "innobackupex will not attempt to kill any queries.", - (uchar*) &opt_ibx_kill_long_queries_timeout, - (uchar*) &opt_ibx_kill_long_queries_timeout, 0, GET_UINT, + "Old disabled option which has no effect anymore (not needed " + "with BACKUP LOCKS)", + (uchar*) 0, (uchar*) 0, 0, GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"ftwrl-wait-timeout", OPT_LOCK_WAIT_TIMEOUT, - "This option specifies time in seconds that innobackupex should wait " - "for queries that would block FTWRL before running it. If there are " - "still such queries when the timeout expires, innobackupex terminates " - "with an error. Default is 0, in which case innobackupex does not " - "wait for queries to complete and starts FTWRL immediately.", - (uchar*) &opt_ibx_lock_wait_timeout, - (uchar*) &opt_ibx_lock_wait_timeout, 0, GET_UINT, + "Alias for startup-wait-timeout", + (uchar*) &opt_ibx_lock_wait_timeout, + (uchar*) &opt_ibx_lock_wait_timeout, 0, GET_UINT, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"startup-wait-timeout", OPT_LOCK_WAIT_TIMEOUT, + "This option specifies time in seconds that mariadb-backup should wait for " + "BACKUP STAGE START to complete. BACKUP STAGE START has to wait until all " + "currently running queries using explicite LOCK TABLES has ended. " + "If there are still such queries when the timeout expires, mariadb-backup " + "terminates with an error. Default is 0, in which case mariadb-backup waits " + "indefinitely for BACKUP STAGE START to finish", + (uchar*) &opt_ibx_lock_wait_timeout, + (uchar*) &opt_ibx_lock_wait_timeout, 0, GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"ftwrl-wait-threshold", OPT_LOCK_WAIT_THRESHOLD, - "This option specifies the query run time threshold which is used by " - "innobackupex to detect long-running queries with a non-zero value " - "of --ftwrl-wait-timeout. FTWRL is not started until such " - "long-running queries exist. This option has no effect if " - "--ftwrl-wait-timeout is 0. Default value is 60 seconds.", - (uchar*) &opt_ibx_lock_wait_threshold, - (uchar*) &opt_ibx_lock_wait_threshold, 0, GET_UINT, + "Old disabled option which has no effect anymore (not needed " + "with BACKUP LOCKS)", + (uchar*) 0, (uchar*) 0, 0, GET_UINT, REQUIRED_ARG, 60, 0, 0, 0, 0, 0}, {"safe-slave-backup-timeout", OPT_SAFE_SLAVE_BACKUP_TIMEOUT, @@ -864,10 +850,8 @@ ibx_init() opt_slave_info = opt_ibx_slave_info; opt_no_lock = opt_ibx_no_lock; opt_safe_slave_backup = opt_ibx_safe_slave_backup; - opt_rsync = opt_ibx_rsync; opt_force_non_empty_dirs = opt_ibx_force_non_empty_dirs; opt_noversioncheck = opt_ibx_noversioncheck; - opt_no_backup_locks = opt_ibx_no_backup_locks; opt_decompress = opt_ibx_decompress; opt_incremental_history_name = opt_ibx_incremental_history_name; diff --git a/extra/mariabackup/read_filt.cc b/extra/mariabackup/read_filt.cc index 58920055..c7c0aa55 100644 --- a/extra/mariabackup/read_filt.cc +++ b/extra/mariabackup/read_filt.cc @@ -32,29 +32,13 @@ Perform read filter context initialization that is common to all read filters. */ static void -common_init( -/*========*/ +rf_pass_through_init( xb_read_filt_ctxt_t* ctxt, /*!offset = 0; ctxt->data_file_size = cursor->statinfo.st_size; ctxt->buffer_capacity = cursor->buf_size; - ctxt->page_size = cursor->page_size; -} - -/****************************************************************//** -Initialize the pass-through read filter. */ -static -void -rf_pass_through_init( -/*=================*/ - xb_read_filt_ctxt_t* ctxt, /*!offset; *read_batch_len = ctxt->data_file_size - ctxt->offset; - if (*read_batch_len > (ib_int64_t)ctxt->buffer_capacity) { - *read_batch_len = ctxt->buffer_capacity; - } - - ctxt->offset += *read_batch_len; -} - -/****************************************************************//** -Deinitialize the pass-through read filter. */ -static -void -rf_pass_through_deinit( -/*===================*/ - xb_read_filt_ctxt_t* ctxt __attribute__((unused))) - /*!bitmap_range = xb_page_bitmap_range_init(changed_page_bitmap, - space_id); - ctxt->filter_batch_end = 0; -} - -/****************************************************************//** -Get the next batch of pages for the bitmap read filter. */ -static -void -rf_bitmap_get_next_batch( -/*=====================*/ - xb_read_filt_ctxt_t* ctxt, /*!page_size; - - start_page_id = (ulint)(ctxt->offset / page_size); - - xb_a (ctxt->offset % page_size == 0); - - if (start_page_id == ctxt->filter_batch_end) { - - /* Used up all the previous bitmap range, get some more */ - ulint next_page_id; - - /* Find the next changed page using the bitmap */ - next_page_id = xb_page_bitmap_range_get_next_bit - (ctxt->bitmap_range, TRUE); - - if (next_page_id == ULINT_UNDEFINED) { - *read_batch_len = 0; - return; - } - - ctxt->offset = next_page_id * page_size; - - /* Find the end of the current changed page block by searching - for the next cleared bitmap bit */ - ctxt->filter_batch_end - = xb_page_bitmap_range_get_next_bit(ctxt->bitmap_range, - FALSE); - xb_a(next_page_id < ctxt->filter_batch_end); - } - - *read_batch_start = ctxt->offset; - if (ctxt->filter_batch_end == ULINT_UNDEFINED) { - /* No more cleared bits in the bitmap, need to copy all the - remaining pages. */ - *read_batch_len = ctxt->data_file_size - ctxt->offset; - } else { - *read_batch_len = ctxt->filter_batch_end * page_size - - ctxt->offset; - } - - /* If the page block is larger than the buffer capacity, limit it to - buffer capacity. The subsequent invocations will continue returning - the current block in buffer-sized pieces until ctxt->filter_batch_end - is reached, trigerring the next bitmap query. */ - if (*read_batch_len > (ib_int64_t)ctxt->buffer_capacity) { + if (*read_batch_len > (int64_t)ctxt->buffer_capacity) { *read_batch_len = ctxt->buffer_capacity; } ctxt->offset += *read_batch_len; - xb_a (ctxt->offset % page_size == 0); - xb_a (*read_batch_start % page_size == 0); - xb_a (*read_batch_len % page_size == 0); -} - -/****************************************************************//** -Deinitialize the changed page bitmap-based read filter. */ -static -void -rf_bitmap_deinit( -/*=============*/ - xb_read_filt_ctxt_t* ctxt) /*!bitmap_range); } /* The pass-through read filter */ xb_read_filt_t rf_pass_through = { &rf_pass_through_init, &rf_pass_through_get_next_batch, - &rf_pass_through_deinit -}; - -/* The changed page bitmap-based read filter */ -xb_read_filt_t rf_bitmap = { - &rf_bitmap_init, - &rf_bitmap_get_next_batch, - &rf_bitmap_deinit }; diff --git a/extra/mariabackup/read_filt.h b/extra/mariabackup/read_filt.h index 51150705..caf8ac56 100644 --- a/extra/mariabackup/read_filt.h +++ b/extra/mariabackup/read_filt.h @@ -25,42 +25,27 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA #ifndef XB_READ_FILT_H #define XB_READ_FILT_H -#include "changed_page_bitmap.h" - -typedef uint32_t space_id_t; +#include +#include struct xb_fil_cur_t; /* The read filter context */ struct xb_read_filt_ctxt_t { - ib_int64_t offset; /*!< current file offset */ - ib_int64_t data_file_size; /*!< data file size */ + int64_t offset; /*!< current file offset */ + int64_t data_file_size; /*!< data file size */ size_t buffer_capacity;/*!< read buffer capacity */ - space_id_t space_id; /*!< space id */ - /* The following fields used only in bitmap filter */ - /* Move these to union if any other filters are added in future */ - xb_page_bitmap_range *bitmap_range; /*!< changed page bitmap range - iterator for space_id */ - ulint page_size; /*!< page size */ - ulint filter_batch_end;/*!< the ending page id of the - current changed page block in - the bitmap */ - /** TODO: remove this default constructor */ - xb_read_filt_ctxt_t() : page_size(0) {} }; /* The read filter */ struct xb_read_filt_t { void (*init)(xb_read_filt_ctxt_t* ctxt, - const xb_fil_cur_t* cursor, - ulint space_id); + const xb_fil_cur_t* cursor); void (*get_next_batch)(xb_read_filt_ctxt_t* ctxt, - ib_int64_t* read_batch_start, - ib_int64_t* read_batch_len); - void (*deinit)(xb_read_filt_ctxt_t* ctxt); + int64_t* read_batch_start, + int64_t* read_batch_len); }; extern xb_read_filt_t rf_pass_through; -extern xb_read_filt_t rf_bitmap; #endif diff --git a/extra/mariabackup/thread_pool.cc b/extra/mariabackup/thread_pool.cc new file mode 100644 index 00000000..e18581f4 --- /dev/null +++ b/extra/mariabackup/thread_pool.cc @@ -0,0 +1,50 @@ +#include "thread_pool.h" +#include "common.h" + +bool ThreadPool::start(size_t threads_count) { + if (!m_stopped) + return false; + m_stopped = false; + for (unsigned i = 0; i < threads_count; ++i) + m_threads.emplace_back(&ThreadPool::thread_func, this, i); + return true; +} + +void ThreadPool::stop() { + if (m_stopped) + return; + m_stop = true; + m_cv.notify_all(); + for (auto &t : m_threads) + t.join(); + m_stopped = true; +}; + +void ThreadPool::push(ThreadPool::job_t &&j) { + std::unique_lock lock(m_mutex); + m_jobs.push(j); + lock.unlock(); + m_cv.notify_one(); +} + +void ThreadPool::thread_func(unsigned thread_num) { + if (my_thread_init()) + die("Can't init mysql thread"); + std::unique_lock lock(m_mutex); + while(true) { + if (m_stop) + goto exit; + while (!m_jobs.empty()) { + if (m_stop) + goto exit; + job_t j = std::move(m_jobs.front()); + m_jobs.pop(); + lock.unlock(); + j(thread_num); + lock.lock(); + } + m_cv.wait(lock, [&] { return m_stop || !m_jobs.empty(); }); + } +exit: + my_thread_end(); +} diff --git a/extra/mariabackup/thread_pool.h b/extra/mariabackup/thread_pool.h new file mode 100644 index 00000000..10ad74c6 --- /dev/null +++ b/extra/mariabackup/thread_pool.h @@ -0,0 +1,62 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include "trx0sys.h" + +class ThreadPool { +public: + typedef std::function job_t; + + ThreadPool() { m_stop = false; m_stopped = true; } + ThreadPool (ThreadPool &&other) = delete; + ThreadPool & operator= (ThreadPool &&other) = delete; + ThreadPool(const ThreadPool &) = delete; + ThreadPool & operator= (const ThreadPool &) = delete; + + bool start(size_t threads_count); + void stop(); + void push(job_t &&j); + size_t threads_count() const { return m_threads.size(); } +private: + void thread_func(unsigned thread_num); + std::mutex m_mutex; + std::condition_variable m_cv; + std::queue m_jobs; + std::atomic m_stop; + std::atomic m_stopped; + std::vector m_threads; +}; + +class TasksGroup { +public: + TasksGroup(ThreadPool &thread_pool) : m_thread_pool(thread_pool) { + m_tasks_count = 0; + m_tasks_result = 1; + } + void push_task(ThreadPool::job_t &&j) { + ++m_tasks_count; + m_thread_pool.push(std::forward(j)); + } + void finish_task(int res) { + --m_tasks_count; + m_tasks_result.fetch_and(res); + } + int get_result() const { return m_tasks_result; } + bool is_finished() const { + return !m_tasks_count; + } + bool wait_for_finish() { + while (!is_finished()) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + return get_result(); + } +private: + ThreadPool &m_thread_pool; + std::atomic m_tasks_count; + std::atomic m_tasks_result; +}; diff --git a/extra/mariabackup/write_filt.cc b/extra/mariabackup/write_filt.cc index 052cea26..13f19ca6 100644 --- a/extra/mariabackup/write_filt.cc +++ b/extra/mariabackup/write_filt.cc @@ -144,6 +144,18 @@ wf_incremental_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile) return false; } + /* Check whether TRX_SYS page has been changed */ + if (mach_read_from_4(page + FIL_PAGE_SPACE_ID) + == TRX_SYS_SPACE + && mach_read_from_4(page + FIL_PAGE_OFFSET) + == TRX_SYS_PAGE_NO) { + msg(cursor->thread_n, + "--incremental backup is impossible if " + "the server had been restarted with " + "different innodb_undo_tablespaces."); + return false; + } + /* updated page */ if (cp->npages == page_size / 4) { /* flush buffer */ diff --git a/extra/mariabackup/wsrep.cc b/extra/mariabackup/wsrep.cc index acaf5c50..15463a85 100644 --- a/extra/mariabackup/wsrep.cc +++ b/extra/mariabackup/wsrep.cc @@ -55,6 +55,9 @@ permission notice: #define XB_GALERA_INFO_FILENAME "xtrabackup_galera_info" #define XB_GALERA_DONOR_INFO_FILENAME "donor_galera_info" +/* backup copy of galera info file as sent by donor */ +#define XB_GALERA_INFO_FILENAME_SST "xtrabackup_galera_info_SST" + /*********************************************************************** Store Galera checkpoint info in the 'xtrabackup_galera_info' file, if that information is present in the trx system header. Otherwise, do nothing. */ @@ -68,21 +71,47 @@ xb_write_galera_info(bool incremental_prepare) long long seqno; MY_STAT statinfo; - /* Do not overwrite an existing file to be compatible with - servers with older server versions */ - if (!incremental_prepare && - my_stat(XB_GALERA_INFO_FILENAME, &statinfo, MYF(0)) != NULL) { - - return; - } - xid.null(); + /* try to read last wsrep XID from innodb rsegs, we will use it + instead of galera info file received from donor + */ if (!trx_rseg_read_wsrep_checkpoint(xid)) { - + /* no worries yet, SST may have brought in galera info file + from some old MariaDB version, which does not support + wsrep XID storing in innodb rsegs + */ return; } + /* if SST brought in galera info file, copy it as *_SST file + this will not be used, saved just for future reference + */ + if (my_stat(XB_GALERA_INFO_FILENAME, &statinfo, MYF(0)) != NULL) { + FILE* fp_in = fopen(XB_GALERA_INFO_FILENAME, "r"); + FILE* fp_out = fopen(XB_GALERA_INFO_FILENAME_SST, "w"); + + char buf[BUFSIZ] = {'\0'}; + size_t size; + while ((size = fread(buf, 1, BUFSIZ, fp_in))) { + if (fwrite(buf, 1, size, fp_out) != strlen(buf)) { + die( + "could not write to " + XB_GALERA_INFO_FILENAME_SST + ", errno = %d\n", + errno); + } + } + if (!feof(fp_in)) { + die( + XB_GALERA_INFO_FILENAME_SST + " not fully copied\n" + ); + } + fclose(fp_out); + fclose(fp_in); + } + wsrep_uuid_t uuid; memcpy(uuid.data, wsrep_xid_uuid(&xid), sizeof(uuid.data)); if (wsrep_uuid_print(&uuid, uuid_str, @@ -97,7 +126,6 @@ xb_write_galera_info(bool incremental_prepare) "could not create " XB_GALERA_INFO_FILENAME ", errno = %d\n", errno); - exit(EXIT_FAILURE); } seqno = wsrep_xid_seqno(&xid); diff --git a/extra/mariabackup/xb_plugin.cc b/extra/mariabackup/xb_plugin.cc deleted file mode 100644 index 7470d376..00000000 --- a/extra/mariabackup/xb_plugin.cc +++ /dev/null @@ -1,229 +0,0 @@ -/* Copyright (c) 2017, 2022, MariaDB Corporation. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -extern struct st_maria_plugin *mysql_optional_plugins[]; -extern struct st_maria_plugin *mysql_mandatory_plugins[]; -static void xb_plugin_init(int argc, char **argv); - -extern char *xb_plugin_load; -extern char *xb_plugin_dir; - -const int PLUGIN_MAX_ARGS = 1024; -std::vector backup_plugins_args; - -const char *QUERY_PLUGIN = -"SELECT plugin_name, plugin_library, @@plugin_dir" -" FROM information_schema.plugins WHERE plugin_type='ENCRYPTION'" -" OR (plugin_type = 'DAEMON' AND plugin_name LIKE 'provider\\_%')" -" AND plugin_status='ACTIVE'"; - -std::string xb_plugin_config; - -static void add_to_plugin_load_list(const char *plugin_def) -{ - opt_plugin_load_list_ptr->push_back(new i_string(plugin_def)); -} - -static char XTRABACKUP_EXE[] = "xtrabackup"; - -/* - Read "plugin-load" value from backup-my.cnf during prepare phase. - The value is stored during backup phase. -*/ -static std::string get_plugin_from_cnf(const char *dir) -{ - std::string path = dir + std::string("/backup-my.cnf"); - FILE *f = fopen(path.c_str(), "r"); - if (!f) - { - die("Can't open %s for reading", path.c_str()); - } - char line[512]; - std::string plugin_load; - while (fgets(line, sizeof(line), f)) - { - if (strncmp(line, "plugin_load=", 12) == 0) - { - plugin_load = line + 12; - // remote \n at the end of string - plugin_load.resize(plugin_load.size() - 1); - break; - } - } - fclose(f); - return plugin_load; -} - - -void xb_plugin_backup_init(MYSQL *mysql) -{ - MYSQL_RES *result; - MYSQL_ROW row; - std::ostringstream oss; - char *argv[PLUGIN_MAX_ARGS]; - char show_query[1024] = ""; - std::string plugin_load; - int argc; - - result = xb_mysql_query(mysql, QUERY_PLUGIN, true, true); - while ((row = mysql_fetch_row(result))) - { - char *name= row[0]; - char *library= row[1]; - char *dir= row[2]; - - if (!plugin_load.length()) - { -#ifdef _WIN32 - for (char *p = dir; *p; p++) - if (*p == '\\') *p = '/'; -#endif - strncpy(opt_plugin_dir, dir, FN_REFLEN - 1); - opt_plugin_dir[FN_REFLEN - 1] = '\0'; - oss << "plugin_dir=" << '"' << dir << '"' << std::endl; - } - - plugin_load += std::string(";") + name; - - if (library) - { - /* Remove shared library suffixes, in case we'll prepare on different OS.*/ - const char *extensions[] = { ".dll", ".so", 0 }; - for (size_t i = 0; extensions[i]; i++) - { - const char *ext = extensions[i]; - if (ends_with(library, ext)) - library[strlen(library) - strlen(ext)] = 0; - } - plugin_load += std::string("=") + library; - } - - if (strncmp(name, "provider_", 9) == 0) - continue; - - /* Read plugin variables. */ - snprintf(show_query, sizeof(show_query), "SHOW variables like '%s_%%'", name); - } - mysql_free_result(result); - if (!plugin_load.length()) - return; - - oss << "plugin_load=" << plugin_load.c_str() + 1 << std::endl; - - /* Required to load the plugin later.*/ - add_to_plugin_load_list(plugin_load.c_str() + 1); - - - if (*show_query) - { - result = xb_mysql_query(mysql, show_query, true, true); - while ((row = mysql_fetch_row(result))) - { - std::string arg("--"); - arg += row[0]; - arg += "="; - arg += row[1]; - backup_plugins_args.push_back(arg); - oss << row[0] << "=" << row[1] << std::endl; - } - - mysql_free_result(result); - - /* Check whether to encrypt logs. */ - result = xb_mysql_query(mysql, "select @@innodb_encrypt_log", true, true); - row = mysql_fetch_row(result); - srv_encrypt_log = (row != 0 && row[0][0] == '1'); - oss << "innodb_encrypt_log=" << row[0] << std::endl; - - mysql_free_result(result); - } - - xb_plugin_config = oss.str(); - - argc = 0; - argv[argc++] = XTRABACKUP_EXE; - for(size_t i = 0; i < backup_plugins_args.size(); i++) - { - argv[argc++] = (char *)backup_plugins_args[i].c_str(); - if (argc == PLUGIN_MAX_ARGS - 2) - break; - } - argv[argc] = 0; - - xb_plugin_init(argc, argv); -} - -const char *xb_plugin_get_config() -{ - return xb_plugin_config.c_str(); -} - -extern int finalize_encryption_plugin(st_plugin_int *plugin); - - -void xb_plugin_prepare_init(int argc, char **argv, const char *dir) -{ - std::string plugin_load= get_plugin_from_cnf(dir ? dir : "."); - if (plugin_load.size()) - { - msg("Loading plugins from %s", plugin_load.c_str()); - } - else - { - finalize_encryption_plugin(0); - return; - } - - add_to_plugin_load_list(plugin_load.c_str()); - - if (xb_plugin_dir) - { - strncpy(opt_plugin_dir, xb_plugin_dir, FN_REFLEN - 1); - opt_plugin_dir[FN_REFLEN - 1] = '\0'; - } - - char **new_argv = new char *[argc + 2]; - new_argv[0] = XTRABACKUP_EXE; - memcpy(&new_argv[1], argv, argc*sizeof(char *)); - - xb_plugin_init(argc+1, new_argv); - - delete[] new_argv; -} - -static void xb_plugin_init(int argc, char **argv) -{ - /* Patch optional and mandatory plugins, we only need to load the one in xb_plugin_load. */ - mysql_optional_plugins[0] = mysql_mandatory_plugins[0] = 0; - plugin_maturity = MariaDB_PLUGIN_MATURITY_UNKNOWN; /* mariabackup accepts all plugins */ - msg("Loading plugins"); - for (int i= 1; i < argc; i++) - msg("\t Plugin parameter : '%s'", argv[i]); - plugin_init(&argc, argv, PLUGIN_INIT_SKIP_PLUGIN_TABLE); -} - diff --git a/extra/mariabackup/xb_plugin.h b/extra/mariabackup/xb_plugin.h deleted file mode 100644 index fea24b6b..00000000 --- a/extra/mariabackup/xb_plugin.h +++ /dev/null @@ -1,5 +0,0 @@ -#include -#include -extern void xb_plugin_backup_init(MYSQL *mysql); -extern const char* xb_plugin_get_config(); -extern void xb_plugin_prepare_init(int argc, char **argv, const char *dir); diff --git a/extra/mariabackup/xbstream.cc b/extra/mariabackup/xbstream.cc index 6306806b..d69a0029 100644 --- a/extra/mariabackup/xbstream.cc +++ b/extra/mariabackup/xbstream.cc @@ -262,7 +262,7 @@ mode_create(int argc, char **argv) return 1; } - stream = xb_stream_write_new(); + stream = xb_stream_write_new(nullptr, nullptr); if (stream == NULL) { msg("%s: xb_stream_write_new() failed.", my_progname); return 1; @@ -287,7 +287,7 @@ mode_create(int argc, char **argv) goto err; } - file = xb_stream_write_open(stream, filepath, &mystat, NULL, NULL); + file = xb_stream_write_open(stream, filepath, &mystat, false); if (file == NULL) { goto err; } @@ -314,7 +314,8 @@ err: static file_entry_t * -file_entry_new(extract_ctxt_t *ctxt, const char *path, uint pathlen) +file_entry_new(extract_ctxt_t *ctxt, const char *path, uint pathlen, + uchar chunk_flags) { file_entry_t *entry; ds_file_t *file; @@ -331,7 +332,8 @@ file_entry_new(extract_ctxt_t *ctxt, const char *path, uint pathlen) } entry->pathlen = pathlen; - file = ds_open(ctxt->ds_ctxt, path, NULL); + file = ds_open(ctxt->ds_ctxt, path, NULL, + chunk_flags == XB_STREAM_FLAG_REWRITE); if (file == NULL) { msg("%s: failed to create file.", my_progname); @@ -412,10 +414,50 @@ extract_worker_thread_func(void *arg) (uchar *) chunk.path, chunk.pathlen); + if (entry && (chunk.type == XB_CHUNK_TYPE_REMOVE || + chunk.type == XB_CHUNK_TYPE_RENAME)) { + msg("%s: rename and remove chunks can not be applied to opened file: %s", + my_progname, chunk.path); + pthread_mutex_unlock(ctxt->mutex); + break; + } + + if (chunk.type == XB_CHUNK_TYPE_REMOVE) { + if (ds_remove(ctxt->ds_ctxt, chunk.path)) { + msg("%s: error on file removing: %s", my_progname, chunk.path); + pthread_mutex_unlock(ctxt->mutex); + res = XB_STREAM_READ_ERROR; + break; + } + pthread_mutex_unlock(ctxt->mutex); + continue; + } + + if (chunk.type == XB_CHUNK_TYPE_RENAME) { + if (my_hash_search(ctxt->filehash, + reinterpret_cast(chunk.data), chunk.length)) { + msg("%s: rename chunks can not be applied to opened file: %s", + my_progname, reinterpret_cast(chunk.data)); + pthread_mutex_unlock(ctxt->mutex); + break; + } + if (ds_rename(ctxt->ds_ctxt, chunk.path, + reinterpret_cast(chunk.data))) { + msg("%s: error on file renaming: %s to %s", my_progname, + reinterpret_cast(chunk.data), chunk.path); + pthread_mutex_unlock(ctxt->mutex); + res = XB_STREAM_READ_ERROR; + break; + } + pthread_mutex_unlock(ctxt->mutex); + continue; + } + if (entry == NULL) { entry = file_entry_new(ctxt, chunk.path, - chunk.pathlen); + chunk.pathlen, + chunk.flags); if (entry == NULL) { pthread_mutex_unlock(ctxt->mutex); break; @@ -432,6 +474,18 @@ extract_worker_thread_func(void *arg) pthread_mutex_unlock(ctxt->mutex); + if (chunk.type == XB_CHUNK_TYPE_SEEK) { + if (ds_seek_set(entry->file, chunk.offset)) { + msg("%s: my_seek() failed.", my_progname); + pthread_mutex_unlock(&entry->mutex); + res = XB_STREAM_READ_ERROR; + break; + } + entry->offset = chunk.offset; + pthread_mutex_unlock(&entry->mutex); + continue; + } + res = xb_stream_validate_checksum(&chunk); if (res != XB_STREAM_READ_CHUNK) { diff --git a/extra/mariabackup/xbstream.h b/extra/mariabackup/xbstream.h index 1b36ec24..c8b2997d 100644 --- a/extra/mariabackup/xbstream.h +++ b/extra/mariabackup/xbstream.h @@ -29,6 +29,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA /* Chunk flags */ /* Chunk can be ignored if unknown version/format */ #define XB_STREAM_FLAG_IGNORABLE 0x01 +#define XB_STREAM_FLAG_REWRITE 0x02 /* Magic + flags + type + path len */ #define CHUNK_HEADER_CONSTANT_LEN ((sizeof(XB_STREAM_CHUNK_MAGIC) - 1) + \ @@ -48,18 +49,21 @@ typedef enum { /************************************************************************ Write interface. */ -typedef ssize_t xb_stream_write_callback(xb_wstream_file_t *file, +typedef ssize_t xb_stream_write_callback( void *userdata, const void *buf, size_t len); -xb_wstream_t *xb_stream_write_new(void); - +xb_wstream_t *xb_stream_write_new( + xb_stream_write_callback *write_callback, void *user_data); xb_wstream_file_t *xb_stream_write_open(xb_wstream_t *stream, const char *path, - MY_STAT *mystat, void *userdata, - xb_stream_write_callback *onwrite); + const MY_STAT *mystat, bool rewrite); int xb_stream_write_data(xb_wstream_file_t *file, const void *buf, size_t len); - +int xb_stream_write_seek_set(xb_wstream_file_t *file, my_off_t offset); +int xb_stream_write_remove(xb_wstream_t *stream, const char *path); +int +xb_stream_write_rename( + xb_wstream_t *stream, const char *old_path, const char *new_path); int xb_stream_write_close(xb_wstream_file_t *file); int xb_stream_write_done(xb_wstream_t *stream); @@ -76,6 +80,9 @@ typedef enum { typedef enum { XB_CHUNK_TYPE_UNKNOWN = '\0', XB_CHUNK_TYPE_PAYLOAD = 'P', + XB_CHUNK_TYPE_RENAME = 'R', + XB_CHUNK_TYPE_REMOVE = 'D', + XB_CHUNK_TYPE_SEEK = 'S', XB_CHUNK_TYPE_EOF = 'E' } xb_chunk_type_t; diff --git a/extra/mariabackup/xbstream_read.cc b/extra/mariabackup/xbstream_read.cc index b54a9815..d82176ad 100644 --- a/extra/mariabackup/xbstream_read.cc +++ b/extra/mariabackup/xbstream_read.cc @@ -59,6 +59,9 @@ validate_chunk_type(uchar code) { switch ((xb_chunk_type_t) code) { case XB_CHUNK_TYPE_PAYLOAD: + case XB_CHUNK_TYPE_RENAME: + case XB_CHUNK_TYPE_REMOVE: + case XB_CHUNK_TYPE_SEEK: case XB_CHUNK_TYPE_EOF: return (xb_chunk_type_t) code; default: @@ -159,57 +162,91 @@ xb_stream_read_chunk(xb_rstream_t *stream, xb_rstream_chunk_t *chunk) } chunk->path[pathlen] = '\0'; - if (chunk->type == XB_CHUNK_TYPE_EOF) { + if (chunk->type == XB_CHUNK_TYPE_EOF || + chunk->type == XB_CHUNK_TYPE_REMOVE) { return XB_STREAM_READ_CHUNK; } - /* Payload length */ - F_READ(tmpbuf, 16); - ullval = uint8korr(tmpbuf); - if (ullval > (ulonglong) SIZE_T_MAX) { - msg("xb_stream_read_chunk(): chunk length is too large at " - "offset 0x%llx: 0x%llx.", (ulonglong) stream->offset, - ullval); - goto err; + if (chunk->type == XB_CHUNK_TYPE_RENAME) { + F_READ(tmpbuf, 4); + size_t new_pathlen = uint4korr(tmpbuf); + if (new_pathlen >= FN_REFLEN) { + msg("xb_stream_read_chunk(): path length (%lu) for new name of 'rename'" + " chunk is too large", (ulong) new_pathlen); + goto err; + } + chunk->length = new_pathlen; + stream->offset +=4; } - chunk->length = (size_t) ullval; - stream->offset += 8; - - /* Payload offset */ - ullval = uint8korr(tmpbuf + 8); - if (ullval > (ulonglong) MY_OFF_T_MAX) { - msg("xb_stream_read_chunk(): chunk offset is too large at " - "offset 0x%llx: 0x%llx.", (ulonglong) stream->offset, - ullval); - goto err; + else if (chunk->type == XB_CHUNK_TYPE_SEEK) { + F_READ(tmpbuf, 8); + chunk->offset = uint8korr(tmpbuf); + stream->offset += 8; + return XB_STREAM_READ_CHUNK; + } + else { + /* Payload length */ + F_READ(tmpbuf, 16); + ullval = uint8korr(tmpbuf); + if (ullval > (ulonglong) SIZE_T_MAX) { + msg("xb_stream_read_chunk(): chunk length is too large at " + "offset 0x%llx: 0x%llx.", (ulonglong) stream->offset, + ullval); + goto err; + } + chunk->length = (size_t) ullval; + stream->offset += 8; + + /* Payload offset */ + ullval = uint8korr(tmpbuf + 8); + if (ullval > (ulonglong) MY_OFF_T_MAX) { + msg("xb_stream_read_chunk(): chunk offset is too large at " + "offset 0x%llx: 0x%llx.", (ulonglong) stream->offset, + ullval); + goto err; + } + chunk->offset = (my_off_t) ullval; + stream->offset += 8; } - chunk->offset = (my_off_t) ullval; - stream->offset += 8; - /* Reallocate the buffer if needed */ - if (chunk->length > chunk->buflen) { - chunk->data = my_realloc(PSI_NOT_INSTRUMENTED, chunk->data, chunk->length, - MYF(MY_WME | MY_ALLOW_ZERO_PTR)); + /* Reallocate the buffer if needed, take into account trailing '\0' for + new file name in the case of XB_CHUNK_TYPE_RENAME */ + if (chunk->length + 1 > chunk->buflen) { + chunk->data = my_realloc(PSI_NOT_INSTRUMENTED, chunk->data, + chunk->length + 1, MYF(MY_WME | MY_ALLOW_ZERO_PTR)); if (chunk->data == NULL) { msg("xb_stream_read_chunk(): failed to increase buffer " - "to %lu bytes.", (ulong) chunk->length); + "to %lu bytes.", (ulong) chunk->length + 1); goto err; } - chunk->buflen = chunk->length; + chunk->buflen = chunk->length + 1; } - /* Checksum */ - F_READ(tmpbuf, 4); - chunk->checksum = uint4korr(tmpbuf); - chunk->checksum_offset = stream->offset; - - /* Payload */ - if (chunk->length > 0) { + if (chunk->type == XB_CHUNK_TYPE_RENAME) { + if (chunk->length == 0) { + msg("xb_stream_read_chunk(): failed to read new name for file to rename " + ": %s", chunk->path); + goto err; + } F_READ(chunk->data, chunk->length); stream->offset += chunk->length; + reinterpret_cast(chunk->data)[chunk->length] = '\0'; + ++chunk->length; } + else { + /* Checksum */ + F_READ(tmpbuf, 4); + chunk->checksum = uint4korr(tmpbuf); + chunk->checksum_offset = stream->offset; + + /* Payload */ + if (chunk->length > 0) { + F_READ(chunk->data, chunk->length); + stream->offset += chunk->length; + } - stream->offset += 4; + stream->offset += 4; + } return XB_STREAM_READ_CHUNK; diff --git a/extra/mariabackup/xbstream_write.cc b/extra/mariabackup/xbstream_write.cc index 5801e867..926e091b 100644 --- a/extra/mariabackup/xbstream_write.cc +++ b/extra/mariabackup/xbstream_write.cc @@ -21,6 +21,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA #include #include #include +#include #include "common.h" #include "xbstream.h" @@ -29,6 +30,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA struct xb_wstream_struct { pthread_mutex_t mutex; + xb_stream_write_callback *write; + void *user_data; }; struct xb_wstream_file_struct { @@ -39,8 +42,7 @@ struct xb_wstream_file_struct { char *chunk_ptr; size_t chunk_free; my_off_t offset; - void *userdata; - xb_stream_write_callback *write; + bool rewrite; }; static int xb_stream_flush(xb_wstream_file_t *file); @@ -50,7 +52,7 @@ static int xb_stream_write_eof(xb_wstream_file_t *file); static ssize_t -xb_stream_default_write_callback(xb_wstream_file_t *file __attribute__((unused)), +xb_stream_default_write_callback( void *userdata __attribute__((unused)), const void *buf, size_t len) { @@ -60,21 +62,31 @@ xb_stream_default_write_callback(xb_wstream_file_t *file __attribute__((unused)) } xb_wstream_t * -xb_stream_write_new(void) +xb_stream_write_new( + xb_stream_write_callback *write_callback, void *user_data) { xb_wstream_t *stream; stream = (xb_wstream_t *) my_malloc(PSI_NOT_INSTRUMENTED, sizeof(xb_wstream_t), MYF(MY_FAE)); pthread_mutex_init(&stream->mutex, NULL); + if (write_callback) { +#ifdef _WIN32 + setmode(fileno(stdout), _O_BINARY); +#endif + stream->write = write_callback; + stream->user_data = user_data; + } + else { + stream->write = xb_stream_default_write_callback; + stream->user_data = user_data; + } return stream;; } xb_wstream_file_t * xb_stream_write_open(xb_wstream_t *stream, const char *path, - MY_STAT *mystat __attribute__((unused)), - void *userdata, - xb_stream_write_callback *onwrite) + const MY_STAT *mystat __attribute__((unused)), bool rewrite) { xb_wstream_file_t *file; size_t path_len; @@ -109,16 +121,7 @@ xb_stream_write_open(xb_wstream_t *stream, const char *path, file->offset = 0; file->chunk_ptr = file->chunk; file->chunk_free = XB_STREAM_MIN_CHUNK_SIZE; - if (onwrite) { -#ifdef _WIN32 - setmode(fileno(stdout), _O_BINARY); -#endif - file->userdata = userdata; - file->write = onwrite; - } else { - file->userdata = NULL; - file->write = xb_stream_default_write_callback; - } + file->rewrite = rewrite; return file; } @@ -202,7 +205,8 @@ xb_stream_write_chunk(xb_wstream_file_t *file, const void *buf, size_t len) memcpy(ptr, XB_STREAM_CHUNK_MAGIC, sizeof(XB_STREAM_CHUNK_MAGIC) - 1); ptr += sizeof(XB_STREAM_CHUNK_MAGIC) - 1; - *ptr++ = 0; /* Chunk flags */ + *ptr++ = + file->rewrite ? XB_STREAM_FLAG_REWRITE : 0; /* Chunk flags */ *ptr++ = (uchar) XB_CHUNK_TYPE_PAYLOAD; /* Chunk type */ @@ -227,11 +231,11 @@ xb_stream_write_chunk(xb_wstream_file_t *file, const void *buf, size_t len) xb_ad(ptr <= tmpbuf + sizeof(tmpbuf)); - if (file->write(file, file->userdata, tmpbuf, ptr-tmpbuf) == -1) + if (stream->write(stream->user_data, tmpbuf, ptr-tmpbuf) == -1) goto err; - if (file->write(file, file->userdata, buf, len) == -1) /* Payload */ + if (stream->write(stream->user_data, buf, len) == -1) /* Payload */ goto err; file->offset+= len; @@ -247,6 +251,38 @@ err: return 1; } +int xb_stream_write_seek_set(xb_wstream_file_t *file, my_off_t offset) +{ + /* Chunk magic + flags + chunk type + path_len + path + offset */ + uchar tmpbuf[sizeof(XB_STREAM_CHUNK_MAGIC) - 1 + 1 + 1 + 4 + + FN_REFLEN + 8]; + int error = 0; + xb_wstream_t *stream = file->stream; + uchar *ptr = tmpbuf; + /* Chunk magic */ + memcpy(ptr, XB_STREAM_CHUNK_MAGIC, sizeof(XB_STREAM_CHUNK_MAGIC) - 1); + ptr += sizeof(XB_STREAM_CHUNK_MAGIC) - 1; + *ptr++ = 0; /* Chunk flags */ + *ptr++ = (uchar) XB_CHUNK_TYPE_SEEK; /* Chunk type */ + int4store(ptr, file->path_len); /* Path length */ + ptr += 4; + memcpy(ptr, file->path, file->path_len); /* Path */ + ptr += file->path_len; + int8store(ptr, static_cast(offset)); /* Offset */ + ptr += 8; + if (xb_stream_flush(file)) + return 1; + pthread_mutex_lock(&stream->mutex); + if (stream->write(stream->user_data, tmpbuf, ptr-tmpbuf) == -1) + error = 1; + if (!error) + file->offset = offset; + pthread_mutex_unlock(&stream->mutex); + if (xb_stream_flush(file)) + return 1; + return error; +} + static int xb_stream_write_eof(xb_wstream_file_t *file) @@ -278,7 +314,7 @@ xb_stream_write_eof(xb_wstream_file_t *file) xb_ad(ptr <= tmpbuf + sizeof(tmpbuf)); - if (file->write(file, file->userdata, tmpbuf, + if (stream->write(stream->user_data, tmpbuf, (ulonglong) (ptr - tmpbuf)) == -1) goto err; @@ -291,3 +327,77 @@ err: return 1; } + + +int +xb_stream_write_remove(xb_wstream_t *stream, const char *path) { + /* Chunk magic + flags + chunk type + path_len + path */ + uchar tmpbuf[sizeof(XB_STREAM_CHUNK_MAGIC) - 1 + 1 + 1 + 4 + FN_REFLEN]; + uchar *ptr = tmpbuf; + /* Chunk magic */ + memcpy(ptr, XB_STREAM_CHUNK_MAGIC, sizeof(XB_STREAM_CHUNK_MAGIC) - 1); + ptr += sizeof(XB_STREAM_CHUNK_MAGIC) - 1; + + *ptr++ = 0; /* Chunk flags */ + + *ptr++ = (uchar) XB_CHUNK_TYPE_REMOVE; /* Chunk type */ + size_t path_len = strlen(path); + int4store(ptr, path_len); /* Path length */ + ptr += 4; + + memcpy(ptr, path, path_len); /* Path */ + ptr += path_len; + + xb_ad(ptr <= tmpbuf + sizeof(tmpbuf)); + + pthread_mutex_lock(&stream->mutex); + + ssize_t result = stream->write(stream->user_data, tmpbuf, + (ulonglong) (ptr - tmpbuf)); + + pthread_mutex_unlock(&stream->mutex); + + return result < 0; + +} + +int +xb_stream_write_rename( + xb_wstream_t *stream, const char *old_path, const char *new_path) { + /* Chunk magic + flags + chunk type + path_len + path + path_len + path*/ + uchar tmpbuf[sizeof(XB_STREAM_CHUNK_MAGIC) - 1 + 1 + 1 + + 4 + FN_REFLEN + 4 + FN_REFLEN]; + uchar *ptr = tmpbuf; + /* Chunk magic */ + memcpy(ptr, XB_STREAM_CHUNK_MAGIC, sizeof(XB_STREAM_CHUNK_MAGIC) - 1); + ptr += sizeof(XB_STREAM_CHUNK_MAGIC) - 1; + + *ptr++ = 0; /* Chunk flags */ + + *ptr++ = (uchar) XB_CHUNK_TYPE_RENAME; /* Chunk type */ + size_t path_len = strlen(old_path); + int4store(ptr, path_len); /* Path length */ + ptr += 4; + + memcpy(ptr, old_path, path_len); /* Path */ + ptr += path_len; + + path_len = strlen(new_path); + int4store(ptr, path_len); /* Path length */ + ptr += 4; + + memcpy(ptr, new_path, path_len); /* Path */ + ptr += path_len; + + xb_ad(ptr <= tmpbuf + sizeof(tmpbuf)); + + pthread_mutex_lock(&stream->mutex); + + ssize_t result = stream->write(stream->user_data, tmpbuf, + (ulonglong) (ptr - tmpbuf)); + + pthread_mutex_unlock(&stream->mutex); + + return result < 0; +} + diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 485cb143..5979bbd3 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -4,7 +4,7 @@ MariaBackup: hot backup tool for InnoDB Originally Created 3/3/2009 Yasufumi Kinoshita Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. -(c) 2017, 2022, MariaDB Corporation. +(c) 2017, 2024, MariaDB Corporation. Portions written by Marko Mäkelä. This program is free software; you can redistribute it and/or modify @@ -54,7 +54,6 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA #include #include -#include #include #ifdef __linux__ @@ -70,6 +69,7 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA # include #endif +#include "aria_backup_client.h" #include #include @@ -82,6 +82,7 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA #include #include #include "ha_innodb.h" +#include "fts0types.h" #include #include @@ -97,23 +98,26 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA #include "xb_regex.h" #include "fil_cur.h" #include "write_filt.h" -#include "xtrabackup.h" #include "ds_buffer.h" #include "ds_tmpfile.h" #include "xbstream.h" -#include "changed_page_bitmap.h" #include "read_filt.h" #include "backup_wsrep.h" #include "innobackupex.h" #include "backup_mysql.h" #include "backup_copy.h" #include "backup_mysql.h" -#include "xb_plugin.h" +#include "encryption_plugin.h" #include #include #include #include #include +#include +#include "ddl_log.h" +#include "common_engine.h" +#include "lex_string.h" +#include "sql_table.h" #include "backup_debug.h" #define MB_CORRUPTED_PAGES_FILE "innodb_corrupted_pages" @@ -126,6 +130,9 @@ int sd_notifyf() { return 0; } int sys_var_init(); +extern const char* fts_common_tables[]; +extern const fts_index_selector_t fts_index_selector[]; + /* === xtrabackup specific options === */ #define DEFAULT_TARGET_DIR "./xtrabackup_backupfiles/" char xtrabackup_real_target_dir[FN_REFLEN] = DEFAULT_TARGET_DIR; @@ -140,8 +147,8 @@ my_bool xtrabackup_decrypt_decompress; my_bool xtrabackup_print_param; my_bool xtrabackup_mysqld_args; my_bool xtrabackup_help; - my_bool xtrabackup_export; +my_bool ignored_option; longlong xtrabackup_use_memory; @@ -155,7 +162,6 @@ char *xtrabackup_incremental; lsn_t incremental_lsn; lsn_t incremental_to_lsn; lsn_t incremental_last_lsn; -xb_page_bitmap *changed_page_bitmap; char *xtrabackup_incremental_basedir; /* for --backup */ char *xtrabackup_extra_lsndir; /* for --backup with --extra-lsndir */ @@ -195,10 +201,12 @@ struct xb_filter_entry_t{ xb_filter_entry_t *name_hash; }; +lsn_t checkpoint_lsn_start; +lsn_t checkpoint_no_start; /** whether log_copying_thread() is active; protected by recv_sys.mutex */ static bool log_copying_running; -int xtrabackup_parallel; +uint xtrabackup_parallel; char *xtrabackup_stream_str = NULL; xb_stream_fmt_t xtrabackup_stream_fmt = XB_STREAM_FMT_NONE; @@ -255,7 +263,7 @@ recv_sys.mutex. */ static std::set fail_undo_ids; longlong innobase_page_size = (1LL << 14); /* 16KB */ -char* innobase_buffer_pool_filename = NULL; +char *innobase_buffer_pool_filename = NULL; /* The default values for the following char* start-up parameters are determined in innobase_init below: */ @@ -361,10 +369,8 @@ my_bool opt_galera_info = FALSE; my_bool opt_slave_info = FALSE; my_bool opt_no_lock = FALSE; my_bool opt_safe_slave_backup = FALSE; -my_bool opt_rsync = FALSE; my_bool opt_force_non_empty_dirs = FALSE; my_bool opt_noversioncheck = FALSE; -my_bool opt_no_backup_locks = FALSE; my_bool opt_decompress = FALSE; my_bool opt_remove_original; my_bool opt_log_innodb_page_corruption; @@ -422,6 +428,8 @@ pthread_cond_t scanned_lsn_cond; /** Store the deferred tablespace name during --backup */ static std::set defer_space_names; +typedef decltype(fil_space_t::id) space_id_t; + typedef std::map space_id_to_name_t; struct ddl_tracker_t { @@ -696,8 +704,190 @@ typedef void (*process_single_tablespace_func_t)(const char *dirname, uint32_t defer_space_id); static dberr_t enumerate_ibd_files(process_single_tablespace_func_t callback); +const char *convert_dst(const char *dst) { + return + (xtrabackup_copy_back || xtrabackup_move_back) ? + dst : trim_dotslash(dst); +} + +std::string convert_tablename_to_filepath( + const char *data_dir_path, const std::string &db, const std::string &table) { + char dbbuff[FN_REFLEN]; + char tbbuff[FN_REFLEN]; + (void)tablename_to_filename(db.c_str(), dbbuff, sizeof(dbbuff)); + (void)tablename_to_filename(table.c_str(), tbbuff, sizeof(tbbuff)); + std::string result(data_dir_path); + result.append(1, FN_LIBCHAR).append(dbbuff). + append(1, FN_LIBCHAR).append(tbbuff); + return result; +} + +std::tuple +convert_filepath_to_tablename(const char *filepath) { + char db_name_orig[FN_REFLEN]; + char table_name_orig[FN_REFLEN]; + parse_db_table_from_file_path(filepath, db_name_orig, table_name_orig); + if (!db_name_orig[0] || !table_name_orig[0]) + return std::make_tuple("", "", ""); + char db_name_conv[FN_REFLEN]; + char table_name_conv[FN_REFLEN]; + filename_to_tablename(db_name_orig, db_name_conv, sizeof(db_name_conv)); + filename_to_tablename( + table_name_orig, table_name_conv, sizeof(table_name_conv)); + if (!db_name_conv[0] || !table_name_conv[0]) + return std::make_tuple("", "", ""); + return std::make_tuple(db_name_conv, table_name_conv, + std::string(db_name_orig).append("/").append(table_name_orig)); +} + +std::string get_table_version_from_image(const std::vector &frm_image) { + DBUG_ASSERT(frm_image.size() >= 64); + + if (!strncmp((char*) frm_image.data(), "TYPE=VIEW\n", 10)) + return {}; + + if (!is_binary_frm_header(frm_image.data())) + return {}; + + /* Length of the MariaDB extra2 segment in the form file. */ + uint len = uint2korr(frm_image.data() + 4); + const uchar *extra2= frm_image.data() + 64; + + if (*extra2 == '/') // old frm had '/' there + return {}; + + const uchar *e2end= extra2 + len; + while (extra2 + 3 <= e2end) + { + uchar type= *extra2++; + size_t length= *extra2++; + if (!length) + { + if (extra2 + 2 >= e2end) + return {}; + length= uint2korr(extra2); + extra2+= 2; + if (length < 256) + return {}; + } + if (extra2 + length > e2end) + return {}; + if (type == EXTRA2_TABLEDEF_VERSION) { + char buff[MY_UUID_STRING_LENGTH]; + my_uuid2str(extra2, buff, 1); + return std::string(buff, buff + MY_UUID_STRING_LENGTH); + } + extra2+= length; + } + + return {}; +} + +std::pair + get_table_engine_from_image(const std::vector &frm_image) { + + DBUG_ASSERT(frm_image.size() >= 64); + + if (!strncmp((char*) frm_image.data(), "TYPE=VIEW\n", 10)) + return std::make_pair(false, DB_TYPE_UNKNOWN); + + if (!is_binary_frm_header(frm_image.data())) + return std::make_pair(false, DB_TYPE_UNKNOWN); + + legacy_db_type dbt = (legacy_db_type)frm_image[3]; + + if (dbt >= DB_TYPE_FIRST_DYNAMIC) + return std::make_pair(false, DB_TYPE_UNKNOWN); + + if (dbt != DB_TYPE_PARTITION_DB) + return std::make_pair(false, dbt); + + dbt = (legacy_db_type)frm_image[61]; + return std::make_pair(true, + dbt < DB_TYPE_FIRST_DYNAMIC ? dbt : DB_TYPE_UNKNOWN); +} + +std::vector read_frm_image(File file) { + std::vector frm_image; + MY_STAT state; + + if (mysql_file_fstat(file, &state, MYF(MY_WME))) + return frm_image; + + frm_image.resize((size_t)state.st_size, 0); + + if (mysql_file_read( + file, frm_image.data(), (size_t)state.st_size, MYF(MY_NABP))) + frm_image.clear(); + + return frm_image; +} + +std::string read_table_version_id(File file) { + auto frm_image = read_frm_image(file); + if (frm_image.empty()) + return {}; + return get_table_version_from_image(frm_image); +} + +bool is_log_table(const char *dbname, const char *tablename) { + DBUG_ASSERT(dbname); + DBUG_ASSERT(tablename); + + LEX_CSTRING lex_db; + LEX_CSTRING lex_table; + lex_db.str = dbname; + lex_db.length = strlen(dbname); + lex_table.str = tablename; + lex_table.length = strlen(tablename); + + if (!lex_string_eq(&MYSQL_SCHEMA_NAME, &lex_db)) + return false; + + if (lex_string_eq(&GENERAL_LOG_NAME, &lex_table)) + return true; + + if (lex_string_eq(&SLOW_LOG_NAME, &lex_table)) + return true; + + return false; +} + +bool is_stats_table(const char *dbname, const char *tablename) { + DBUG_ASSERT(dbname); + DBUG_ASSERT(tablename); + + LEX_CSTRING lex_db; + LEX_CSTRING lex_table; + lex_db.str = dbname; + lex_db.length = strlen(dbname); + lex_table.str = tablename; + lex_table.length = strlen(tablename); + + if (!lex_string_eq(&MYSQL_SCHEMA_NAME, &lex_db)) + return false; + + CHARSET_INFO *ci= system_charset_info; + + return (lex_table.length > 4 && + /* one of mysql.*_stat tables, but not mysql.innodb* tables*/ + ((my_tolower(ci, lex_table.str[lex_table.length-5]) == 's' && + my_tolower(ci, lex_table.str[lex_table.length-4]) == 't' && + my_tolower(ci, lex_table.str[lex_table.length-3]) == 'a' && + my_tolower(ci, lex_table.str[lex_table.length-2]) == 't' && + my_tolower(ci, lex_table.str[lex_table.length-1]) == 's') && + !(my_tolower(ci, lex_table.str[0]) == 'i' && + my_tolower(ci, lex_table.str[1]) == 'n' && + my_tolower(ci, lex_table.str[2]) == 'n' && + my_tolower(ci, lex_table.str[3]) == 'o'))); +} + /* ======== Datafiles iterator ======== */ struct datafiles_iter_t { + datafiles_iter_t() : space(fil_system.space_list.end()), node(nullptr), started(FALSE) { + } + ~datafiles_iter_t() { + } space_list_t::iterator space = fil_system.space_list.end(); fil_node_t *node = nullptr; bool started = false; @@ -777,8 +967,6 @@ static void *dbug_execute_in_new_connection(void *arg) return nullptr; } -static pthread_t dbug_alter_thread; - /* Execute query from a new connection, in own thread. @@ -789,8 +977,9 @@ Execute query from a new connection, in own thread. otherwise query should return error. @param expected_errno - if not 0, and query finished with error, expected mysql_errno() +@return created thread id */ -static void dbug_start_query_thread( +static pthread_t dbug_start_query_thread( const char *query, const char *wait_state, int expected_err, @@ -802,12 +991,14 @@ static void dbug_start_query_thread( par->expect_err = expected_err; par->expect_errno = expected_errno; par->con = xb_mysql_connect(); - - mysql_thread_create(0, &dbug_alter_thread, nullptr, + if (mysql_set_server_option(par->con, MYSQL_OPTION_MULTI_STATEMENTS_ON)) + die("Can't set multistatement option for query: %s", query); + pthread_t result_thread; + mysql_thread_create(0, &result_thread, nullptr, dbug_execute_in_new_connection, par); if (!wait_state) - return; + return result_thread; char q[256]; snprintf(q, sizeof(q), @@ -829,7 +1020,11 @@ static void dbug_start_query_thread( end: msg("query '%s' on connection %lu reached state '%s'", query, mysql_thread_id(par->con), wait_state); + return result_thread; } + +static pthread_t dbug_alter_thread; +static pthread_t dbug_emulate_ddl_on_intermediate_table_thread; #endif void mdl_lock_all() @@ -952,6 +1147,31 @@ static void backup_file_op(uint32_t space_id, int type, } } +static bool check_if_fts_table(const char *file_name) { + const char *table_name_start = strrchr(file_name, '/'); + if (table_name_start) + ++table_name_start; + else + table_name_start = file_name; + + if (!starts_with(table_name_start,"FTS_")) + return false; + + const char *table_name_end = strrchr(table_name_start, '.'); + if (!table_name_end) + table_name_end = table_name_start + strlen(table_name_start); + ptrdiff_t table_name_len = table_name_end - table_name_end; + + for (const char **suffix = fts_common_tables; *suffix; ++suffix) + if (!strncmp(table_name_start, *suffix, table_name_len)) + return true; + for (size_t i = 0; fts_index_selector[i].suffix; ++i) + if (!strncmp(table_name_start, fts_index_selector[i].suffix, + table_name_len)) + return true; + + return false; +} /* This callback is called if DDL operation is detected, @@ -985,8 +1205,9 @@ static void backup_file_op_fail(uint32_t space_id, int type, break; case FILE_DELETE: fail = !check_if_skip_table( - filename_to_spacename(name, len).c_str()); - msg("DDL tracking : delete %u \"%.*s\"", space_id, int(len), name); + filename_to_spacename(name, len).c_str()) + && !check_if_fts_table(reinterpret_cast(name)); + msg("DDL tracking : delete %u \"%.*s\"", space_id, int(len), name); break; default: ut_ad(0); @@ -1112,6 +1333,7 @@ enum options_xtrabackup OPT_INNODB_LOG_FILE_BUFFERING, #endif OPT_INNODB_LOG_FILE_SIZE, + OPT_INNODB_LOG_FILES_IN_GROUP, OPT_INNODB_OPEN_FILES, OPT_XTRA_DEBUG_SYNC, OPT_INNODB_CHECKSUM_ALGORITHM, @@ -1129,9 +1351,9 @@ enum options_xtrabackup OPT_NO_LOCK, OPT_SAFE_SLAVE_BACKUP, OPT_RSYNC, + OPT_NO_BACKUP_LOCKS, OPT_FORCE_NON_EMPTY_DIRS, OPT_NO_VERSION_CHECK, - OPT_NO_BACKUP_LOCKS, OPT_DECOMPRESS, OPT_INCREMENTAL_HISTORY_NAME, OPT_INCREMENTAL_HISTORY_UUID, @@ -1343,8 +1565,10 @@ struct my_option xb_client_options[]= { 0, 0, 0, 0, 0, 0}, {"no-lock", OPT_NO_LOCK, - "Use this option to disable table lock " - "with \"FLUSH TABLES WITH READ LOCK\". Use it only if ALL your " + "This option should not be used as " + "mariadb-backup now is using BACKUP LOCKS, which minimizes the " + "lock time. ALTER TABLE can run in parallel with BACKUP LOCKS." + "Use the --no-lock option it only if ALL your " "tables are InnoDB and you DO NOT CARE about the binary log " "position of the backup. This option shouldn't be used if there " "are any DDL statements being executed or if any updates are " @@ -1373,14 +1597,12 @@ struct my_option xb_client_options[]= { GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"rsync", OPT_RSYNC, - "Uses the rsync utility to optimize local file " - "transfers. When this option is specified, " XB_TOOL_NAME " uses rsync " - "to copy all non-InnoDB files instead of spawning a separate cp for " - "each file, which can be much faster for servers with a large number " - "of databases or tables. This option cannot be used together with " - "--stream.", - (uchar *) &opt_rsync, (uchar *) &opt_rsync, 0, GET_BOOL, NO_ARG, 0, 0, 0, - 0, 0, 0}, + "Obsolete depricated option", + &ignored_option, &ignored_option, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"no-backup-locks", OPT_NO_BACKUP_LOCKS, + "Obsolete depricated option", + &ignored_option, &ignored_option, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"force-non-empty-directories", OPT_FORCE_NON_EMPTY_DIRS, "This " @@ -1398,15 +1620,6 @@ struct my_option xb_client_options[]= { (uchar *) &opt_noversioncheck, (uchar *) &opt_noversioncheck, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"no-backup-locks", OPT_NO_BACKUP_LOCKS, - "This option controls if " - "backup locks should be used instead of FLUSH TABLES WITH READ LOCK " - "on the backup stage. The option has no effect when backup locks are " - "not supported by the server. This option is enabled by default, " - "disable with --no-backup-locks.", - (uchar *) &opt_no_backup_locks, (uchar *) &opt_no_backup_locks, 0, - GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"decompress", OPT_DECOMPRESS, "Decompresses all files with the .qp " "extension in a backup previously made with the --compress option. " @@ -1483,11 +1696,10 @@ struct my_option xb_client_options[]= { (uchar *) &opt_remove_original, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"ftwrl-wait-query-type", OPT_LOCK_WAIT_QUERY_TYPE, - "This option specifies which types of queries are allowed to complete " - "before " XB_TOOL_NAME " will issue the global lock. Default is all.", - (uchar *) &opt_lock_wait_query_type, (uchar *) &opt_lock_wait_query_type, - &query_type_typelib, GET_ENUM, REQUIRED_ARG, QUERY_TYPE_ALL, 0, 0, 0, 0, - 0}, + "Old disabled option which has no effect anymore (not needed " + "with BACKUP LOCKS)", + (uchar*) 0, (uchar*) 0, &query_type_typelib, GET_ENUM, + REQUIRED_ARG, QUERY_TYPE_ALL, 0, 0, 0, 0, 0}, {"kill-long-query-type", OPT_KILL_LONG_QUERY_TYPE, "This option specifies which types of queries should be killed to " @@ -1504,32 +1716,31 @@ struct my_option xb_client_options[]= { NULL, NULL, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, {"kill-long-queries-timeout", OPT_KILL_LONG_QUERIES_TIMEOUT, - "This option specifies the number of seconds " XB_TOOL_NAME " waits " - "between starting FLUSH TABLES WITH READ LOCK and killing those " - "queries that block it. Default is 0 seconds, which means " - XB_TOOL_NAME " will not attempt to kill any queries.", - (uchar *) &opt_kill_long_queries_timeout, - (uchar *) &opt_kill_long_queries_timeout, 0, GET_UINT, REQUIRED_ARG, 0, 0, + "Old disabled option which has no effect anymore (not needed " + "with BACKUP LOCKS)", + (uchar*) 0, (uchar*) 0, 0, GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"ftwrl-wait-timeout", OPT_LOCK_WAIT_TIMEOUT, - "This option specifies time in seconds that " XB_TOOL_NAME " should wait " - "for queries that would block FTWRL before running it. If there are " - "still such queries when the timeout expires, " XB_TOOL_NAME " terminates " - "with an error. Default is 0, in which case " XB_TOOL_NAME " does not " - "wait for queries to complete and starts FTWRL immediately.", - (uchar *) &opt_lock_wait_timeout, (uchar *) &opt_lock_wait_timeout, 0, - GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + "Alias for startup-wait-timeout", + (uchar*) &opt_lock_wait_timeout, (uchar*) &opt_lock_wait_timeout, + 0, GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"startup-wait-timeout", OPT_LOCK_WAIT_TIMEOUT, + "This option specifies time in seconds that mariadb-backup should wait for " + "BACKUP STAGE START to complete. BACKUP STAGE START has to wait until all " + "currently running queries using explicite LOCK TABLES has ended. " + "If there are still such queries when the timeout expires, mariadb-backup " + "terminates with an error. Default is 0, in which case mariadb-backup waits " + "indefinitely for BACKUP STAGE START to finish", + (uchar*) &opt_lock_wait_timeout, (uchar*) &opt_lock_wait_timeout, + 0, GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"ftwrl-wait-threshold", OPT_LOCK_WAIT_THRESHOLD, - "This option specifies the query run time threshold which is used by " - XB_TOOL_NAME " to detect long-running queries with a non-zero value " - "of --ftwrl-wait-timeout. FTWRL is not started until such " - "long-running queries exist. This option has no effect if " - "--ftwrl-wait-timeout is 0. Default value is 60 seconds.", - (uchar *) &opt_lock_wait_threshold, (uchar *) &opt_lock_wait_threshold, 0, - GET_UINT, REQUIRED_ARG, 60, 0, 0, 0, 0, 0}, - + "Old disabled option which has no effect anymore (not needed " + "with BACKUP LOCKS)", + (uchar*) 0, (uchar*) 0, 0, GET_UINT, + REQUIRED_ARG, 60, 0, 0, 0, 0, 0}, {"safe-slave-backup-timeout", OPT_SAFE_SLAVE_BACKUP_TIMEOUT, "How many seconds --safe-slave-backup should wait for " @@ -1595,7 +1806,7 @@ struct my_option xb_server_options[] = {"parallel", OPT_XTRA_PARALLEL, "Number of threads to use for parallel datafiles transfer. " "The default value is 1.", - (G_PTR*) &xtrabackup_parallel, (G_PTR*) &xtrabackup_parallel, 0, GET_INT, + (G_PTR*) &xtrabackup_parallel, (G_PTR*) &xtrabackup_parallel, 0, GET_UINT, REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0}, {"extended_validation", OPT_XTRA_EXTENDED_VALIDATION, @@ -1677,8 +1888,8 @@ struct my_option xb_server_options[] = {"innodb_log_buffer_size", OPT_INNODB_LOG_BUFFER_SIZE, "Redo log buffer size in bytes.", (G_PTR*) &log_sys.buf_size, (G_PTR*) &log_sys.buf_size, 0, - IF_WIN(GET_ULL,GET_ULONG), REQUIRED_ARG, 2U << 20, - 2U << 20, SIZE_T_MAX, 0, 4096, 0}, + GET_UINT, REQUIRED_ARG, 2U << 20, + 2U << 20, log_sys.buf_size_max, 0, 4096, 0}, #if defined __linux__ || defined _WIN32 {"innodb_log_file_buffering", OPT_INNODB_LOG_FILE_BUFFERING, "Whether the file system cache for ib_logfile0 is enabled during --backup", @@ -1754,17 +1965,17 @@ struct my_option xb_server_options[] = 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"plugin-dir", OPT_PLUGIN_DIR, - "Server plugin directory. Used to load plugins during 'prepare' phase." - "Has no effect in the 'backup' phase (plugin directory during backup is the same as server's)", - &xb_plugin_dir, &xb_plugin_dir, - 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, + "Server plugin directory. Used to load plugins during 'prepare' phase." + "Has no effect in the 'backup' phase (plugin directory during backup is the same as server's)", + &xb_plugin_dir, &xb_plugin_dir, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, {"aria_log_dir_path", OPT_ARIA_LOG_DIR_PATH, "Path to individual files and their sizes.", &aria_log_dir_path, &aria_log_dir_path, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - {"open_files_limit", OPT_OPEN_FILES_LIMIT, "the maximum number of file " + {"open_files_limit", 0, "the maximum number of file " "descriptors to reserve with setrlimit().", (G_PTR*) &xb_open_files_limit, (G_PTR*) &xb_open_files_limit, 0, GET_ULONG, REQUIRED_ARG, 0, 0, UINT_MAX, 0, 1, 0}, @@ -1885,7 +2096,7 @@ static int prepare_export() IF_WIN("\"","") "\"%s\" --mysqld \"%s\"" " --defaults-extra-file=./backup-my.cnf --defaults-group-suffix=%s --datadir=." " --innodb --innodb-fast-shutdown=0 --loose-partition" - " --innodb-buffer-pool-size=%llu" + " --innodb_purge_rseg_truncate_frequency=1 --innodb-buffer-pool-size=%llu" " --console --skip-log-error --skip-log-bin --bootstrap %s< " BOOTSTRAP_FILENAME IF_WIN("\"",""), mariabackup_exe, @@ -1899,7 +2110,7 @@ static int prepare_export() IF_WIN("\"","") "\"%s\" --mysqld" " --defaults-file=./backup-my.cnf --defaults-group-suffix=%s --datadir=." " --innodb --innodb-fast-shutdown=0 --loose-partition" - " --innodb-buffer-pool-size=%llu" + " --innodb_purge_rseg_truncate_frequency=1 --innodb-buffer-pool-size=%llu" " --console --log-error= --skip-log-bin --bootstrap %s< " BOOTSTRAP_FILENAME IF_WIN("\"",""), mariabackup_exe, @@ -1948,7 +2159,8 @@ static void usage(void) puts("Open source backup tool for InnoDB and XtraDB\n\ \n\ Copyright (C) 2009-2015 Percona LLC and/or its affiliates.\n\ -Portions Copyright (C) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved.\n\ +Portions Copyright (C) 2000, 2011, MySQL AB & Innobase Oy.\n\ +Portions Copyright (C) 2017-2023 MariaDB Corporation / MariaDB Plc.\n\ \n\ This program is free software; you can redistribute it and/or\n\ modify it under the terms of the GNU General Public License\n\ @@ -2021,6 +2233,10 @@ xb_get_one_option(const struct my_option *opt, ADD_PRINT_PARAM_OPT(srv_log_group_home_dir); break; + case OPT_INNODB_LOG_FILES_IN_GROUP: + case OPT_INNODB_LOG_FILE_SIZE: + break; + case OPT_INNODB_FLUSH_METHOD: #ifdef _WIN32 /* From: storage/innobase/handler/ha_innodb.cc:innodb_init_params */ @@ -2133,6 +2349,11 @@ xb_get_one_option(const struct my_option *opt, } } break; + case OPT_RSYNC: + case OPT_NO_BACKUP_LOCKS: + if (my_handle_options_init_variables) + fprintf(stderr, "Obsolete option: %s. Ignored\n", opt->name); + break; #define MYSQL_CLIENT #include "sslopt-case.h" #undef MYSQL_CLIENT @@ -2428,7 +2649,12 @@ static bool innodb_init() os_file_delete_if_exists_func(ib_logfile0.c_str(), nullptr); os_file_t file= os_file_create_func(ib_logfile0.c_str(), OS_FILE_CREATE, OS_FILE_NORMAL, - OS_DATA_FILE_NO_O_DIRECT, false, &ret); +#if defined _WIN32 || defined O_DIRECT + OS_DATA_FILE_NO_O_DIRECT, +#else + OS_DATA_FILE, +#endif + false, &ret); if (!ret) { invalid_log: @@ -3026,12 +3252,7 @@ static my_bool xtrabackup_copy_datafile(ds_ctxt *ds_data, goto skip; } - if (!changed_page_bitmap) { - read_filter = &rf_pass_through; - } - else { - read_filter = &rf_bitmap; - } + read_filter = &rf_pass_through; res = xb_fil_cur_open(&cursor, read_filter, node, thread_n, ULLONG_MAX); if (res == XB_FIL_CUR_SKIP) { @@ -3355,50 +3576,22 @@ To use this facility, you need to 3. start mariabackup with --dbug=+d,debug_mariabackup_events */ void dbug_mariabackup_event(const char *event, - const fil_space_t::name_type key) + const fil_space_t::name_type key, + bool need_lock) { + static std::mutex dbug_mariabackup_event_mutex; char *sql = dbug_mariabackup_get_val(event, key); if (sql && *sql) { msg("dbug_mariabackup_event : executing '%s'", sql); - xb_mysql_query(mysql_connection, sql, false, true); - } + if (need_lock) { + std::lock_guard lock(dbug_mariabackup_event_mutex); + xb_mysql_query(mysql_connection, sql, false, true); + } else + xb_mysql_query(mysql_connection, sql, false, true); + } } #endif // DBUG_OFF -/** Datafiles copying thread.*/ -static void data_copy_thread_func(data_thread_ctxt_t *ctxt) /* thread context */ -{ - uint num = ctxt->num; - fil_node_t* node; - ut_ad(ctxt->corrupted_pages); - - /* - Initialize mysys thread-specific memory so we can - use mysys functions in this thread. - */ - my_thread_init(); - - while ((node = datafiles_iter_next(ctxt->it)) != NULL) { - DBUG_MARIABACKUP_EVENT("before_copy", node->space->name()); - DBUG_EXECUTE_FOR_KEY("wait_innodb_redo_before_copy", - node->space->name(), - backup_wait_for_lsn(get_current_lsn(mysql_connection));); - /* copy the datafile */ - if (xtrabackup_copy_datafile(ctxt->datasinks->m_data, - ctxt->datasinks->m_meta, node, num, NULL, - xtrabackup_incremental ? wf_incremental : wf_write_through, - *ctxt->corrupted_pages)) - die("failed to copy datafile."); - - DBUG_MARIABACKUP_EVENT("after_copy", node->space->name()); - } - - pthread_mutex_lock(ctxt->count_mutex); - (*ctxt->count)--; - pthread_mutex_unlock(ctxt->count_mutex); - - my_thread_end(); -} /************************************************************************ Initialize the appropriate datasink(s). Both local backups and streaming in the @@ -3553,6 +3746,11 @@ static void xb_load_single_table_tablespace(const char *dirname, } if (file->open_read_only(true) != DB_SUCCESS) { + // Ignore FTS tables, as they can be removed for intermediate tables, + // this code must be executed under stronger or equal to BLOCK_DDL lock, + // so there must not be errors for non-intermediate FTS tables. + if (check_if_fts_table(filname)) + return; die("Can't open datafile %s", name); } @@ -4558,7 +4756,6 @@ bool Backup_datasinks::backup_low() if (failed_ids.size() > 0) { return false; } - if (!xtrabackup_incremental) { safe_strcpy(metadata_type, sizeof(metadata_type), "full-backuped"); @@ -4597,16 +4794,441 @@ bool Backup_datasinks::backup_low() return true; } +class InnodbDataCopier { +public: + InnodbDataCopier(Backup_datasinks &backup_datasinks, + CorruptedPages &corrupted_pages, + ThreadPool &thread_pool) : + m_backup_datasinks(backup_datasinks), + m_corrupted_pages(corrupted_pages), + m_tasks(thread_pool) {} + + ~InnodbDataCopier() { + DBUG_ASSERT(m_tasks.is_finished()); + } + + bool start() { + DBUG_ASSERT(m_tasks.is_finished()); + m_tasks.push_task( + std::bind(&InnodbDataCopier::scan_job, this, std::placeholders::_1)); + return true; + } + + bool wait_for_finish() { + return m_tasks.wait_for_finish(); + } + +private: + void scan_job(unsigned thread_num) { + datafiles_iter_t it; + fil_node_t* node; + while ((node = datafiles_iter_next(&it)) != nullptr) { + m_tasks.push_task( + std::bind(&InnodbDataCopier::copy_job, this, node, + std::placeholders::_1)); + } + m_tasks.finish_task(1); + } + + void copy_job(fil_node_t *node, unsigned thread_num) { + DBUG_ASSERT(node); + // TODO: this came from the old code, where it was not thread-safe + // too, use separate mysql connection per thread here + DBUG_MARIABACKUP_EVENT("before_copy", node->space->name()); + DBUG_EXECUTE_FOR_KEY("wait_innodb_redo_before_copy", + node->space->name(), + backup_wait_for_lsn( + get_current_lsn(mysql_connection));); + /* copy the datafile */ + if(xtrabackup_copy_datafile(m_backup_datasinks.m_data, + m_backup_datasinks.m_meta, + node, thread_num, NULL, + xtrabackup_incremental + ? wf_incremental : wf_write_through, + m_corrupted_pages)) + die("mariabackup: Error: failed to copy datafile."); + // TODO: this came from the old code, where it was not thread-safe + // too, use separate mysql connection per thread here + DBUG_MARIABACKUP_EVENT("after_copy", node->space->name()); + m_tasks.finish_task(1); + } + + Backup_datasinks &m_backup_datasinks; + CorruptedPages &m_corrupted_pages; + TasksGroup m_tasks; +}; + + +class BackupStages { + + public: + + BackupStages(ds_ctxt_t *ds_data) : + m_bs_con(nullptr), + m_aria_backup(fil_path_to_mysql_datadir, + aria_log_dir_path, + ds_data, m_con_pool, m_thread_pool), + m_common_backup(fil_path_to_mysql_datadir, ds_data, m_con_pool, + m_thread_pool) {} + + ~BackupStages() { destroy(); } + + bool init() { + if ((m_bs_con = xb_mysql_connect()) == nullptr) + return false; + + while(m_con_pool.size() < xtrabackup_parallel) { + MYSQL *con = xb_mysql_connect(); + if (con == nullptr) + return false; + m_con_pool.push_back(con); + } + + if (!m_thread_pool.start(xtrabackup_parallel)) + return false; + if (!m_aria_backup.init()) + return false; + m_aria_backup.set_post_copy_table_hook( + std::bind(&BackupStages::store_table_version, this, + std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); + m_common_backup.set_post_copy_table_hook( + std::bind(&BackupStages::store_table_version, this, + std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); + return true; + } + + void destroy() { + m_thread_pool.stop(); + while (!m_con_pool.empty()) { + MYSQL *con = m_con_pool.back(); + m_con_pool.pop_back(); + mysql_close(con); + } + if (m_bs_con) + mysql_close(m_bs_con); + m_bs_con = nullptr; + } + + bool stage_start(Backup_datasinks &backup_datasinks, + CorruptedPages &corrupted_pages) { + msg("BACKUP STAGE START"); + if (!opt_no_lock) { + if (opt_safe_slave_backup) { + if (!wait_for_safe_slave(mysql_connection)) { + return(false); + } + } + + history_lock_time = time(NULL); + + if (!lock_for_backup_stage_start(m_bs_con)) { + msg("Error on BACKUP STAGE START query execution"); + return(false); + } + } + + InnodbDataCopier innodb_data_copier(backup_datasinks, + corrupted_pages, + m_thread_pool); + // Start InnoDB data files copy in background + if (!innodb_data_copier.start()) { + msg("Error on starting InnoDB data files backup"); + return false; + } + // Start online non-stats-log Aria tables copying in background + if (!m_aria_backup.start(opt_no_lock)) { + msg("Error on starting Aria data files backup"); + innodb_data_copier.wait_for_finish(); + return false; + } + + // Wait for all innodb data files copy finish + if(!innodb_data_copier.wait_for_finish()) { + msg("InnoDB data files backup process is finished with error"); + return false; + } + // Wait for online non-stats-log Aria tables copy finish + if (!m_aria_backup.wait_for_finish()) { + msg("Aria data files backup process is finished with error"); + return false; + } + + DBUG_MARIABACKUP_EVENT_LOCK("after_aria_background", {}); + + return true; + } + + bool stage_flush() { + msg("BACKUP STAGE FLUSH"); + if (!opt_no_lock && !lock_for_backup_stage_flush(m_bs_con)) { + msg("Error on BACKUP STAGE FLUSH query execution"); + return false; + } + auto tables_in_use = get_tables_in_use(mysql_connection); + // Copy non-stats-log non-in-use tables of non-InnoDB-Aria-RocksDB engines + // in background + if (!m_common_backup.scan(tables_in_use, + &m_copied_common_tables, opt_no_lock, true)) { + msg("Error on scan data directory for common engines"); + return false; + } + // Copy Aria offline non-stats-log non-in-use tables in background + if (!m_aria_backup.copy_offline_tables(&tables_in_use, opt_no_lock, + false)) { + msg("Error on start Aria tables backup"); + return false; + } + + if (!m_aria_backup.copy_log_tail()) { + msg("Error on Aria log tail copy"); + return false; + }; + + // Wait for Aria tables copy finish + if (!m_aria_backup.wait_for_finish()) { + msg("Aria data files backup process is finished with error"); + return false; + } + // Wait for non-InnoDB-Aria-RocksDB engines copy finish + if (!m_common_backup.wait_for_finish()) { + msg("Data files backup process is finished with error"); + return false; + } + + DBUG_EXECUTE_IF("emulate_ddl_on_intermediate_table", + dbug_emulate_ddl_on_intermediate_table_thread = + dbug_start_query_thread( + "SET debug_sync='copy_data_between_tables_after_set_backup_lock " + "SIGNAL copy_started';" + "SET debug_sync='copy_data_between_tables_before_reset_backup_lock " + "SIGNAL before_backup_lock_reset WAIT_FOR backup_lock_reset';" + "SET debug_sync='alter_table_after_temp_table_drop " + "SIGNAL temp_table_dropped';" + "SET SESSION lock_wait_timeout = 1;" + "ALTER TABLE test.t1 ADD COLUMN col1_copy INT, ALGORITHM = COPY;", + NULL, 0, 0); + xb_mysql_query(mysql_connection, + "SET debug_sync='now WAIT_FOR copy_started'", false, true); + ); + + return true; + } + + bool stage_block_ddl(Backup_datasinks &backup_datasinks, + CorruptedPages &corrupted_pages) { + if (!opt_no_lock) { + if (!lock_for_backup_stage_block_ddl(m_bs_con)) { + msg("BACKUP STAGE BLOCK_DDL"); + return false; + } + if (have_galera_enabled) + { + xb_mysql_query(mysql_connection, "SET SESSION wsrep_sync_wait=0", false); + } + } + + ulonglong server_lsn_after_lock = get_current_lsn(mysql_connection); + + // Copy the rest of non-stats-lognon-InnoDB-Aria-RocksDB tables + // Do not execute BACKUP LOCK under BLOCK_DDL stage + if (!m_common_backup.scan(m_copied_common_tables, &m_copied_common_tables, + true, false)) { + msg("Error on scan data directory for common engines"); + return false; + } + // Copy log tables tail + if (!m_common_backup.copy_log_tables(false)) { + msg("Error on copy system tables"); + return false; + } + + // Copy the rest of non-stats Aria tables in background + if (!m_aria_backup.copy_offline_tables(nullptr, true, false)) { + msg("Error on start Aria tables backup"); + return false; + } + + // Copy .frm, .trn and other files + if (!backup_files(backup_datasinks.m_data, + fil_path_to_mysql_datadir)) { + msg("Backup files error"); + return false; + } + + msg("Waiting for log copy thread to read lsn %llu", + server_lsn_after_lock); + backup_wait_for_lsn(server_lsn_after_lock); + corrupted_pages.backup_fix_ddl(backup_datasinks.m_data, + backup_datasinks.m_meta); + + if (!m_aria_backup.copy_log_tail()) { + msg("Error on Aria log tail copy"); + return false; + } + + // Wait for Aria tables copy finish + if (!m_aria_backup.wait_for_finish()) { + msg("Aria data files backup process is finished with error"); + return false; + } + // Wait for non-InnoDB-Aria-RocksDB engines copy finish + if (!m_common_backup.wait_for_finish()) { + msg("Data files backup process is finished with error"); + return false; + } + + ddl_log::backup(fil_path_to_mysql_datadir, + backup_datasinks.m_data, m_tables); + + DBUG_MARIABACKUP_EVENT_LOCK("after_stage_block_ddl", {}); + + return true; + } + + bool stage_block_commit(Backup_datasinks &backup_datasinks) { + msg("BACKUP STAGE BLOCK_COMMIT"); + if (!opt_no_lock && !lock_for_backup_stage_commit(m_bs_con)) { + msg("Error on BACKUP STAGE BLOCK_COMMIT query execution"); + return false; + } + + // Copy log tables tail + if (!m_common_backup.copy_log_tables(true)) { + msg("Error on copy log tables"); + return false; + } + + // Copy stats tables + if (!m_common_backup.copy_stats_tables()) { + msg("Error on copy stats tables"); + return false; + } + + // Copy system Aria files + if (!m_aria_backup.finalize()) { + msg("Error on finalize Aria tables backup"); + return false; + } + + if (!m_common_backup.wait_for_finish()) { + msg("Error on finish common engines backup"); + return false; + } + + if (!m_common_backup.close_log_tables()) { + msg("Error on close log tables"); + return false; + } + + if (!backup_files_from_datadir(backup_datasinks.m_data, + fil_path_to_mysql_datadir, + "aws-kms-key")) { + msg("Error on root data dir files backup"); + return false; + } + + if (has_rocksdb_plugin()) { + rocksdb_create_checkpoint(); + } + + // There is no need to stop slave thread before coping non-Innodb data when + // --no-lock option is used because --no-lock option requires that no DDL or + // DML to non-transaction tables can occur. + if (opt_no_lock) { + if (opt_safe_slave_backup) { + if (!wait_for_safe_slave(mysql_connection)) { + return(false); + } + } + } + + if (opt_slave_info) { + if (!write_slave_info(backup_datasinks.m_data, + mysql_connection)) { + return(false); + } + } + + /* The only reason why Galera/binlog info is written before + wait_for_ibbackup_log_copy_finish() is that after that call the xtrabackup + binary will start streamig a temporary copy of REDO log to stdout and + thus, any streaming from innobackupex would interfere. The only way to + avoid that is to have a single process, i.e. merge innobackupex and + xtrabackup. */ + if (opt_galera_info) { + if (!write_galera_info(backup_datasinks.m_data, + mysql_connection)) { + return(false); + } + } + + bool with_binlogs = opt_binlog_info == BINLOG_INFO_ON; + + if (with_binlogs || opt_galera_info) { + if (!write_binlog_info(backup_datasinks.m_data, + mysql_connection)) { + return(false); + } + } + + if (!opt_no_lock) { + msg("Executing FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS..."); + xb_mysql_query(mysql_connection, + "FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS", false); + } + + return backup_datasinks.backup_low(); + } + + bool stage_end(Backup_datasinks &backup_datasinks) { + msg("BACKUP STAGE END"); + /* release all locks */ + if (!opt_no_lock) { + unlock_all(m_bs_con); + history_lock_time = 0; + } else { + history_lock_time = time(NULL) - history_lock_time; + } + backup_release(); + DBUG_EXECUTE_IF("check_mdl_lock_works", + pthread_join(dbug_alter_thread, nullptr); + ); + + DBUG_EXECUTE_IF("emulate_ddl_on_intermediate_table", + pthread_join( + dbug_emulate_ddl_on_intermediate_table_thread, + nullptr); + ); + + backup_finish(backup_datasinks.m_data); + return true; + } + + void store_table_version( + std::string db, std::string table, std::string table_version) { + auto tk = table_key(db, table); + std::lock_guard lock(m_tables_mutex); + m_tables[std::move(tk)] = std::move(table_version); + } + + private: + Backup_datasinks *backup_datasinks; + MYSQL *m_bs_con; + ThreadPool m_thread_pool; + std::vector m_con_pool; + std::mutex m_tables_mutex; + ddl_log::tables_t m_tables; + aria::Backup m_aria_backup; + common_engine::Backup m_common_backup; + std::unordered_set m_copied_common_tables; +}; + /** Implement --backup @return whether the operation succeeded */ static bool xtrabackup_backup_func() { MY_STAT stat_info; - uint i; - uint count; - pthread_mutex_t count_mutex; CorruptedPages corrupted_pages; - data_thread_ctxt_t *data_threads; Backup_datasinks backup_datasinks; pthread_cond_init(&scanned_lsn_cond, NULL); @@ -4622,7 +5244,7 @@ static bool xtrabackup_backup_func() return(false); } msg("cd to %s", mysql_real_data_home); - xb_plugin_backup_init(mysql_connection); + encryption_plugin_backup_init(mysql_connection); msg("open files limit requested %lu, set to %lu", xb_open_files_limit, xb_set_max_open_files(xb_open_files_limit)); @@ -4663,22 +5285,6 @@ fail: return(false); } - if (srv_buf_pool_size >= 1000 * 1024 * 1024) { - /* Here we still have srv_pool_size counted - in kilobytes (in 4.0 this was in bytes) - srv_boot() converts the value to - pages; if buffer pool is less than 1000 MB, - assume fewer threads. */ - srv_max_n_threads = 50000; - - } else if (srv_buf_pool_size >= 8 * 1024 * 1024) { - - srv_max_n_threads = 10000; - } else { - srv_max_n_threads = 1000; /* saves several MB of memory, - especially in 64-bit - computers */ - } srv_thread_pool_init(); /* Reset the system variables in the recovery module. */ trx_pool_init(); @@ -4698,9 +5304,10 @@ fail: } /* get current checkpoint_lsn */ { + log_sys.latch.wr_lock(SRW_LOCK_CALL); mysql_mutex_lock(&recv_sys.mutex); - dberr_t err = recv_sys.find_checkpoint(); + log_sys.latch.wr_unlock(); if (err != DB_SUCCESS) { msg("Error: cannot read redo log header"); @@ -4796,11 +5403,6 @@ fail: std::thread(log_copying_thread).detach(); - /* FLUSH CHANGED_PAGE_BITMAPS call */ - if (!flush_changed_page_bitmaps()) { - goto fail; - } - ut_a(xtrabackup_parallel > 0); if (xtrabackup_parallel > 1) { @@ -4812,71 +5414,36 @@ fail: mdl_lock_all(); DBUG_EXECUTE_IF("check_mdl_lock_works", - dbug_start_query_thread("ALTER TABLE test.t ADD COLUMN mdl_lock_column int", + dbug_alter_thread = + dbug_start_query_thread("ALTER TABLE test.t ADD COLUMN mdl_lock_column int", "Waiting for table metadata lock", 0, 0);); } - datafiles_iter_t it; - - /* Create data copying threads */ - data_threads = (data_thread_ctxt_t *) - malloc(sizeof(data_thread_ctxt_t) * xtrabackup_parallel); - count = xtrabackup_parallel; - pthread_mutex_init(&count_mutex, NULL); - - for (i = 0; i < (uint) xtrabackup_parallel; i++) { - data_threads[i].it = ⁢ - data_threads[i].num = i+1; - data_threads[i].count = &count; - data_threads[i].count_mutex = &count_mutex; - data_threads[i].corrupted_pages = &corrupted_pages; - data_threads[i].datasinks= &backup_datasinks; - std::thread(data_copy_thread_func, data_threads + i).detach(); - } + BackupStages stages(backup_datasinks.m_data); - /* Wait for threads to exit */ - while (1) { - std::this_thread::sleep_for(std::chrono::seconds(1)); - pthread_mutex_lock(&count_mutex); - bool stop = count == 0; - pthread_mutex_unlock(&count_mutex); - if (stop) { - break; - } - } - - pthread_mutex_destroy(&count_mutex); - free(data_threads); - - DBUG_ASSERT(backup_datasinks.m_data); - DBUG_ASSERT(backup_datasinks.m_meta); - bool ok = backup_start(backup_datasinks.m_data, - backup_datasinks.m_meta, corrupted_pages); + if (!stages.init()) + goto fail; - if (ok) { - ok = backup_datasinks.backup_low(); + if (!stages.stage_start(backup_datasinks, corrupted_pages)) + goto fail; - backup_release(); + if (!stages.stage_flush()) + goto fail; - DBUG_EXECUTE_IF("check_mdl_lock_works", - pthread_join(dbug_alter_thread, nullptr);); + if (!stages.stage_block_ddl(backup_datasinks, corrupted_pages)) + goto fail; - if (ok) { - backup_finish(backup_datasinks.m_data); - } - } + if (!stages.stage_block_commit(backup_datasinks)) + goto fail; - if (opt_log_innodb_page_corruption) - ok = corrupted_pages.print_to_file(backup_datasinks.m_data, - MB_CORRUPTED_PAGES_FILE); + if (!stages.stage_end(backup_datasinks)) + goto fail; - if (!ok) { + if (opt_log_innodb_page_corruption + && !corrupted_pages.print_to_file(backup_datasinks.m_data, + MB_CORRUPTED_PAGES_FILE)) goto fail; - } - if (changed_page_bitmap) { - xb_page_bitmap_deinit(changed_page_bitmap); - } backup_datasinks.destroy(); msg("Redo log (from LSN " LSN_PF " to " LSN_PF ") was copied.", @@ -4938,6 +5505,12 @@ void CorruptedPages::backup_fix_ddl(ds_ctxt *ds_data, ds_ctxt *ds_meta) DBUG_MARIABACKUP_EVENT("backup_fix_ddl", {}); + DBUG_EXECUTE_IF("emulate_ddl_on_intermediate_table", + xb_mysql_query(mysql_connection, + "SET debug_sync='now SIGNAL backup_lock_reset " + "WAIT_FOR temp_table_dropped'", false, true); + ); + for (space_id_to_name_t::iterator iter = ddl_tracker.tables_in_backup.begin(); iter != ddl_tracker.tables_in_backup.end(); iter++) { @@ -5082,6 +5655,7 @@ void CorruptedPages::backup_fix_ddl(ds_ctxt *ds_data, ds_ctxt *ds_meta) } } + /* ================= prepare ================= */ /*********************************************************************** @@ -5568,7 +6142,7 @@ std::string change_extension(std::string filename, std::string new_ext) { } -static void rename_file(const char *from,const char *to) { +void rename_file(const char *from,const char *to) { msg("Renaming %s to %s\n", from, to); if (my_rename(from, to, MY_WME)) { die("Can't rename %s to %s errno %d", from, to, errno); @@ -5590,7 +6164,7 @@ typedef ibool (*handle_datadir_entry_func_t)( void* arg); /*! /dev/null 2>&1") == 0)); - if (!have_rsync) - { - msg("Error: rsync executable not found, cannot run backup with --rsync\n"); - return false; - } - } - n_mixed_options = 0; if (opt_decompress) { @@ -6493,6 +7053,7 @@ xb_init() if (opt_check_privileges && !check_all_privileges()) { return(false); } + history_start_time = time(NULL); } diff --git a/extra/mariabackup/xtrabackup.h b/extra/mariabackup/xtrabackup.h index d091c474..38d7e5fd 100644 --- a/extra/mariabackup/xtrabackup.h +++ b/extra/mariabackup/xtrabackup.h @@ -24,8 +24,15 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA #include #include "datasink.h" #include "xbstream.h" -#include "changed_page_bitmap.h" +#include "fil0fil.h" #include +#include "handler.h" + +#include +#include +#include +#include + #define XB_TOOL_NAME "mariadb-backup" #define XB_HISTORY_TABLE "mysql.mariadb_backup_history" @@ -84,8 +91,6 @@ extern my_bool xb_backup_rocksdb; extern uint opt_protocol; -extern xb_page_bitmap *changed_page_bitmap; - extern char *xtrabackup_incremental; extern my_bool xtrabackup_incremental_force_scan; @@ -112,7 +117,7 @@ extern my_bool xtrabackup_decrypt_decompress; extern char *innobase_data_file_path; extern longlong innobase_page_size; -extern int xtrabackup_parallel; +extern uint xtrabackup_parallel; extern my_bool xb_close_files; extern const char *xtrabackup_compress_alg; @@ -131,7 +136,6 @@ extern my_bool opt_galera_info; extern my_bool opt_slave_info; extern my_bool opt_no_lock; extern my_bool opt_safe_slave_backup; -extern my_bool opt_rsync; extern my_bool opt_force_non_empty_dirs; extern my_bool opt_noversioncheck; extern my_bool opt_no_backup_locks; @@ -288,15 +292,40 @@ fil_file_readdir_next_file( os_file_stat_t* info); /*!< in/out: buffer where the info is returned */ -#ifndef DBUG_OFF -#include -extern void dbug_mariabackup_event(const char *event, - const fil_space_t::name_type key); +const char *convert_dst(const char *dst); -#define DBUG_MARIABACKUP_EVENT(A, B) \ - DBUG_EXECUTE_IF("mariabackup_events", dbug_mariabackup_event(A, B);) -#else -#define DBUG_MARIABACKUP_EVENT(A, B) /* empty */ -#endif // DBUG_OFF +std::string get_table_version_from_image(const std::vector &frm_image); +std::pair + get_table_engine_from_image(const std::vector &frm_image); +std::string read_table_version_id(File file); + +std::string convert_tablename_to_filepath( + const char *data_dir_path, const std::string &db, const std::string &table); + +std::tuple +convert_filepath_to_tablename(const char *filepath); + +typedef std::string table_key_t; + +inline table_key_t table_key(const std::string &db, const std::string &table) { + return std::string(db).append(".").append(table); +}; + +inline table_key_t table_key(const char *db, const char *table) { + return std::string(db).append(".").append(table); +}; + +typedef std::function + post_copy_table_hook_t; + +my_bool +check_if_skip_table( +/******************/ + const char* name); /*!< in: path to the table */ + +bool is_log_table(const char *dbname, const char *tablename); +bool is_stats_table(const char *dbname, const char *tablename); +extern my_bool xtrabackup_copy_back; +extern my_bool xtrabackup_move_back; #endif /* XB_XTRABACKUP_H */ diff --git a/extra/wolfssl/CMakeLists.txt b/extra/wolfssl/CMakeLists.txt index 271e76b8..e3f8da21 100644 --- a/extra/wolfssl/CMakeLists.txt +++ b/extra/wolfssl/CMakeLists.txt @@ -1,86 +1,57 @@ IF(MSVC_INTEL) PROJECT(wolfssl C ASM_MASM) ELSEIF(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64") - PROJECT(wolfssl C ASM) + PROJECT(wolfssl C ASM) ELSE() PROJECT(wolfssl C) ENDIF() IF(CMAKE_SIZEOF_VOID_P MATCHES 8) -IF(MSVC_INTEL) +IF(MSVC_INTEL AND NOT (CMAKE_C_COMPILER_ID MATCHES Clang)) SET(WOLFSSL_INTELASM ON) - SET(WOLFSSL_X86_64_BUILD 1) SET(HAVE_INTEL_RDSEED 1) SET(HAVE_INTEL_RDRAND 1) -ELSEIF(CMAKE_ASM_COMPILER_ID MATCHES "Clang" AND CMAKE_VERSION VERSION_LESS 3.16) - - # WolfSSL 5.5.4 bug workaround below does not work, due to some CMake bug ELSEIF(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64") - SET(WOLFSSL_X86_64_BUILD 1) IF(CMAKE_C_COMPILER_ID MATCHES GNU AND CMAKE_C_COMPILER_VERSION VERSION_LESS 4.9) MESSAGE_ONCE(NO_INTEL_ASSEMBLY "Disable Intel assembly for WolfSSL - compiler is too old") + ELSEIF(WITH_MSAN) + MESSAGE_ONCE(MSAN_CANT_HANDLE_IT "Disable Intel assembly for WolfSSL - MSAN can't handle it") ELSE() - IF(WITH_MSAN) - MESSAGE_ONCE(MSAN_CANT_HANDLE_IT - "Disable Intel assembly for WolfSSL - MSAN can't handle it") - ELSE() - MY_CHECK_C_COMPILER_FLAG(-maes) - MY_CHECK_C_COMPILER_FLAG(-msse4) - MY_CHECK_C_COMPILER_FLAG(-mpclmul) - IF(have_C__maes AND have_C__msse4 AND have_C__mpclmul) - SET(WOLFSSL_INTELASM ON) + MY_CHECK_C_COMPILER_FLAG(-maes) + MY_CHECK_C_COMPILER_FLAG(-msse4) + MY_CHECK_C_COMPILER_FLAG(-mpclmul) + IF(have_C__maes AND have_C__msse4 AND have_C__mpclmul) + SET(WOLFSSL_INTELASM ON) + MY_CHECK_C_COMPILER_FLAG(-mrdrnd) + MY_CHECK_C_COMPILER_FLAG(-mrdseed) + IF(have_C__mrdrnd) + SET(HAVE_INTEL_RDRAND ON) + ENDIF() + IF(have_C__mrdseed) + SET(HAVE_INTEL_RDSEED ON) ENDIF() - ENDIF() - MY_CHECK_C_COMPILER_FLAG(-mrdrnd) - MY_CHECK_C_COMPILER_FLAG(-mrdseed) - IF(have_C__mrdrnd) - SET(HAVE_INTEL_RDRAND ON) - ENDIF() - IF(have_C__mrdseed) - SET(HAVE_INTEL_RDSEED ON) ENDIF() ENDIF() ENDIF() ENDIF() SET(WOLFSSL_SRCDIR ${CMAKE_CURRENT_SOURCE_DIR}/wolfssl/src) +SET(WOLFCRYPT_SRCDIR ${CMAKE_CURRENT_SOURCE_DIR}/wolfssl/wolfcrypt/src) ADD_DEFINITIONS(${SSL_DEFINES}) - -SET(WOLFSSL_SOURCES - ${WOLFSSL_SRCDIR}/crl.c - ${WOLFSSL_SRCDIR}/internal.c - ${WOLFSSL_SRCDIR}/keys.c - ${WOLFSSL_SRCDIR}/tls.c - ${WOLFSSL_SRCDIR}/wolfio.c - ${WOLFSSL_SRCDIR}/ocsp.c - ${WOLFSSL_SRCDIR}/ssl.c - ${WOLFSSL_SRCDIR}/tls13.c) - ADD_DEFINITIONS(-DWOLFSSL_LIB -DBUILDING_WOLFSSL) - +ADD_DEFINITIONS(-DWOLFSSL_SP_4096) INCLUDE_DIRECTORIES(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/wolfssl) -IF(MSVC) - # size_t to long truncation warning - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -wd4267 -wd4334 -wd4028 -wd4244") -ENDIF() - -ADD_CONVENIENCE_LIBRARY(wolfssl ${WOLFSSL_SOURCES}) - -# Workaround linker crash with older Ubuntu binutils -# e.g aborting at ../../bfd/merge.c line 873 in _bfd_merged_section_offset -IF(CMAKE_SYSTEM_NAME MATCHES "Linux") - STRING(REPLACE "-g " "-g1 " CMAKE_C_FLAGS_RELWITHDEBINFO - ${CMAKE_C_FLAGS_RELWITHDEBINFO}) - STRING(REPLACE "-g " "-g1 " CMAKE_C_FLAGS_DEBUG - ${CMAKE_C_FLAGS_DEBUG}) - STRING(REPLACE "-ggdb3 " " " CMAKE_C_FLAGS_RELWITHDEBINFO - ${CMAKE_C_FLAGS_RELWITHDEBINFO}) - STRING(REPLACE "-ggdb3 " " " CMAKE_C_FLAGS_DEBUG - ${CMAKE_C_FLAGS_DEBUG}) -ENDIF() +INCLUDE_DIRECTORIES(${SSL_INCLUDE_DIRS}) -SET(WOLFCRYPT_SRCDIR ${CMAKE_CURRENT_SOURCE_DIR}/wolfssl/wolfcrypt/src) -SET(WOLFCRYPT_SOURCES +add_library(wolfssl STATIC +${WOLFSSL_SRCDIR}/crl.c +${WOLFSSL_SRCDIR}/internal.c +${WOLFSSL_SRCDIR}/keys.c +${WOLFSSL_SRCDIR}/tls.c +${WOLFSSL_SRCDIR}/wolfio.c +${WOLFSSL_SRCDIR}/ocsp.c +${WOLFSSL_SRCDIR}/ssl.c +${WOLFSSL_SRCDIR}/tls13.c ${WOLFCRYPT_SRCDIR}/aes.c ${WOLFCRYPT_SRCDIR}/arc4.c ${WOLFCRYPT_SRCDIR}/asn.c @@ -110,69 +81,56 @@ ${WOLFCRYPT_SRCDIR}/wc_encrypt.c ${WOLFCRYPT_SRCDIR}/hash.c ${WOLFCRYPT_SRCDIR}/wolfmath.c ${WOLFCRYPT_SRCDIR}/kdf.c +${WOLFCRYPT_SRCDIR}/sp_int.c +${WOLFCRYPT_SRCDIR}/sp_c32.c +${WOLFCRYPT_SRCDIR}/sp_c64.c ) -# Use fastmath large number math library. -IF(NOT (MSVC AND CMAKE_C_COMPILER_ID MATCHES Clang)) - # Can't use clang-cl with WOLFSSL_FASTMATH - # due to https://bugs.llvm.org/show_bug.cgi?id=25305 - SET(WOLFSSL_FASTMATH 1) -ENDIF() - -IF(WOLFSSL_FASTMATH) - SET(USE_FAST_MATH 1) - SET(TFM_TIMING_RESISTANT 1) - # FP_MAX_BITS is set high solely to satisfy ssl_8k_key.test - # WolfSSL will use more stack space with it - SET(FP_MAX_BITS 16384) - SET(WOLFCRYPT_SOURCES ${WOLFCRYPT_SOURCES} ${WOLFCRYPT_SRCDIR}/tfm.c) - IF((CMAKE_SIZEOF_VOID_P MATCHES 4) AND (CMAKE_SYSTEM_PROCESSOR MATCHES "86") - AND (NOT MSVC)) - # Workaround https://github.com/wolfSSL/wolfssl/issues/4245 - # On 32bit Intel, to satisfy inline assembly's wish for free registers - # 1. use -fomit-frame-pointer - # 2. With GCC 4, additionally use -fno-PIC, which works on x86 - # (modern GCC has PIC optimizations, that make it unnecessary) - # The following assumes GCC or Clang - SET(TFM_COMPILE_FLAGS "-fomit-frame-pointer") - IF(CMAKE_C_COMPILER_VERSION VERSION_LESS "5") - SET(TFM_COMPILE_FLAGS "${TFM_COMPILE_FLAGS} -fno-PIC") - ENDIF() - SET_SOURCE_FILES_PROPERTIES(${WOLFCRYPT_SRCDIR}/tfm.c - PROPERTIES COMPILE_FLAGS ${TFM_COMPILE_FLAGS}) - ENDIF() -ELSE() - SET(WOLFSSL_SP_MATH_ALL 1) - SET(WOLFCRYPT_SOURCES ${WOLFCRYPT_SOURCES} ${WOLFCRYPT_SRCDIR}/sp_int.c) -ENDIF() - -IF(WOLFSSL_X86_64_BUILD) - LIST(APPEND WOLFCRYPT_SOURCES ${WOLFCRYPT_SRCDIR}/cpuid.c) - IF(MSVC) - SET(WOLFSSL_AESNI 1) - LIST(APPEND WOLFCRYPT_SOURCES - ${WOLFCRYPT_SRCDIR}/aes_asm.asm - ${WOLFCRYPT_SRCDIR}/aes_gcm_asm.asm) - IF(CMAKE_C_COMPILER_ID MATCHES Clang) - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -maes -msse4.2 -mpclmul -mrdrnd -mrdseed") - ENDIF() - ELSEIF(WOLFSSL_INTELASM) - SET(WOLFSSL_AESNI 1) - SET(USE_INTEL_SPEEDUP 1) - LIST(APPEND WOLFCRYPT_SOURCES +# Optimizations, assembly +if(WOLFSSL_INTELASM) + set(WOLFSSL_X86_64_BUILD 1) + set(WOLFSSL_SP_X86_64 1) + set(WOLFSSL_SP_X86_64_ASM 1) + set(WOLFSSL_AESNI 1) + target_sources(wolfssl PRIVATE + ${WOLFCRYPT_SRCDIR}/cpuid.c + ${WOLFCRYPT_SRCDIR}/sp_x86_64.c + ) + if(MSVC_INTEL) + target_sources(wolfssl PRIVATE + ${WOLFCRYPT_SRCDIR}/aes_asm.asm + ${WOLFCRYPT_SRCDIR}/aes_gcm_asm.asm + ${WOLFCRYPT_SRCDIR}/sp_x86_64_asm.asm + ) + target_compile_options(wolfssl PRIVATE + $<$:-maes -msse4.2 -mpclmul -mrdrnd -mrdseed> + $<$:/Zi> + ) + else() + set(USE_INTEL_SPEEDUP 1) + target_sources(wolfssl PRIVATE ${WOLFCRYPT_SRCDIR}/aes_asm.S ${WOLFCRYPT_SRCDIR}/aes_gcm_asm.S ${WOLFCRYPT_SRCDIR}/chacha_asm.S ${WOLFCRYPT_SRCDIR}/poly1305_asm.S ${WOLFCRYPT_SRCDIR}/sha512_asm.S - ${WOLFCRYPT_SRCDIR}/sha256_asm.S) - ADD_DEFINITIONS(-maes -msse4.2 -mpclmul) - # WolfSSL 5.5.4 bug - user_settings.h not included into aes_asm.S - SET_PROPERTY(SOURCE ${WOLFCRYPT_SRCDIR}/aes_asm.S APPEND PROPERTY COMPILE_OPTIONS "-DWOLFSSL_X86_64_BUILD") - ENDIF() -ENDIF() + ${WOLFCRYPT_SRCDIR}/sha256_asm.S + ${WOLFCRYPT_SRCDIR}/sp_x86_64_asm.S + ) + target_compile_options(wolfssl PRIVATE -maes -msse4.2 -mpclmul) + # Workaround 5.5.4 bug (user_settings.h not included into aes_asm.S) + set_property(SOURCE ${WOLFCRYPT_SRCDIR}/aes_asm.S APPEND PROPERTY COMPILE_OPTIONS "-DWOLFSSL_X86_64_BUILD") + endif() +endif() + +# Silence some warnings +if(MSVC) + # truncation warnings + target_compile_options(wolfssl PRIVATE $<$:/wd4244>) + if(CMAKE_C_COMPILER_ID MATCHES Clang) + target_compile_options(wolfssl PRIVATE $<$:-Wno-incompatible-function-pointer-types>) + endif() +endif() CONFIGURE_FILE(user_settings.h.in user_settings.h) -INCLUDE_DIRECTORIES(${SSL_INCLUDE_DIRS}) -ADD_CONVENIENCE_LIBRARY(wolfcrypt ${WOLFCRYPT_SOURCES}) diff --git a/extra/wolfssl/user_settings.h.in b/extra/wolfssl/user_settings.h.in index baa64fcd..489118b3 100644 --- a/extra/wolfssl/user_settings.h.in +++ b/extra/wolfssl/user_settings.h.in @@ -21,6 +21,7 @@ #define HAVE_AESGCM #define HAVE_CHACHA #define HAVE_POLY1305 +#define HAVE_THREAD_LS #define WOLFSSL_AES_COUNTER #define NO_WOLFSSL_STUB #define OPENSSL_ALL @@ -51,20 +52,19 @@ #define NO_RABBIT #define NO_RC4 -/* - FP_MAX_BITS is set high solely to satisfy ssl_8k_key.test - WolfSSL will use more stack space with it, with fastmath -*/ -#cmakedefine FP_MAX_BITS 16384 #define RSA_MAX_SIZE 8192 +#define WOLFSSL_SP_MATH_ALL +#define WOLFSSL_HAVE_SP_RSA +#ifndef WOLFSSL_SP_4096 +#define WOLFSSL_SP_4096 +#endif + #cmakedefine WOLFSSL_AESNI -#cmakedefine USE_FAST_MATH -#cmakedefine TFM_TIMING_RESISTANT #cmakedefine HAVE_INTEL_RDSEED #cmakedefine HAVE_INTEL_RDRAND #cmakedefine USE_INTEL_SPEEDUP -#cmakedefine USE_FAST_MATH #cmakedefine WOLFSSL_X86_64_BUILD -#cmakedefine WOLFSSL_SP_MATH_ALL +#cmakedefine WOLFSSL_SP_X86_64 +#cmakedefine WOLFSSL_SP_X86_64_ASM #endif /* WOLFSSL_USER_SETTINGS_H */ diff --git a/extra/wolfssl/wolfssl/.github/workflows/async.yml b/extra/wolfssl/wolfssl/.github/workflows/async.yml index 84eb4c58..36f50265 100644 --- a/extra/wolfssl/wolfssl/.github/workflows/async.yml +++ b/extra/wolfssl/wolfssl/.github/workflows/async.yml @@ -18,7 +18,7 @@ jobs: # This should be a safe limit for the tests to run. timeout-minutes: 6 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 name: Checkout wolfSSL - name: Test wolfSSL async diff --git a/extra/wolfssl/wolfssl/.github/workflows/curl.yml b/extra/wolfssl/wolfssl/.github/workflows/curl.yml index fe6350e7..69840eb6 100644 --- a/extra/wolfssl/wolfssl/.github/workflows/curl.yml +++ b/extra/wolfssl/wolfssl/.github/workflows/curl.yml @@ -18,11 +18,11 @@ jobs: install: true - name: Upload built lib - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: wolf-install-curl path: build-dir - retention-days: 1 + retention-days: 5 test_curl: name: ${{ matrix.curl_ref }} @@ -38,11 +38,11 @@ jobs: - name: Install test dependencies run: | sudo apt-get update - sudo apt-get install nghttp2 + sudo apt-get install nghttp2 libpsl5 libpsl-dev sudo pip install impacket - name: Download lib - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: wolf-install-curl path: build-dir diff --git a/extra/wolfssl/wolfssl/.github/workflows/docker-Espressif.yml b/extra/wolfssl/wolfssl/.github/workflows/docker-Espressif.yml index 9b9b9be2..4990e928 100644 --- a/extra/wolfssl/wolfssl/.github/workflows/docker-Espressif.yml +++ b/extra/wolfssl/wolfssl/.github/workflows/docker-Espressif.yml @@ -11,7 +11,7 @@ jobs: container: image: espressif/idf:latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Initialize Espressif IDE and build examples run: . /opt/esp/idf/export.sh; IDE/Espressif/ESP-IDF/compileAllExamples.sh espressif_v4_4: @@ -20,7 +20,7 @@ jobs: container: image: espressif/idf:release-v4.4 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Initialize Espressif IDE and build examples run: . /opt/esp/idf/export.sh; IDE/Espressif/ESP-IDF/compileAllExamples.sh espressif_v5_0: @@ -29,6 +29,6 @@ jobs: container: image: espressif/idf:release-v5.0 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Initialize Espressif IDE and build examples run: . /opt/esp/idf/export.sh; IDE/Espressif/ESP-IDF/compileAllExamples.sh diff --git a/extra/wolfssl/wolfssl/.github/workflows/docker-OpenWrt.yml b/extra/wolfssl/wolfssl/.github/workflows/docker-OpenWrt.yml index aa827895..3c348b62 100644 --- a/extra/wolfssl/wolfssl/.github/workflows/docker-OpenWrt.yml +++ b/extra/wolfssl/wolfssl/.github/workflows/docker-OpenWrt.yml @@ -16,15 +16,15 @@ jobs: steps: - name: Install required tools run: apk add argp-standalone asciidoc bash bc binutils bzip2 cdrkit coreutils diffutils elfutils-dev findutils flex musl-fts-dev g++ gawk gcc gettext git grep intltool libxslt linux-headers make musl-libintl musl-obstack-dev ncurses-dev openssl-dev patch perl python3-dev rsync tar unzip util-linux wget zlib-dev autoconf automake libtool - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Compile libwolfssl.so run: ./autogen.sh && ./configure --enable-all && make - name: Upload libwolfssl.so - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: openwrt-libwolfssl.so path: src/.libs/libwolfssl.so - retention-days: 1 + retention-days: 5 compile_container: name: Compile container runs-on: ubuntu-latest @@ -34,16 +34,16 @@ jobs: strategy: fail-fast: false matrix: - release: [ "22.03-SNAPSHOT", "21.02-SNAPSHOT" ] # some other versions: 21.02.0 21.02.5 22.03.0 22.03.3 snapshot + release: [ "22.03.6", "21.02.7" ] # some other versions: 21.02.0 21.02.5 22.03.0 22.03.3 snapshot steps: - - uses: actions/checkout@v3 - - uses: docker/setup-buildx-action@v2 - - uses: actions/download-artifact@v3 + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v3 + - uses: actions/download-artifact@v4 with: name: openwrt-libwolfssl.so path: Docker/OpenWrt/. - name: Build but dont push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: Docker/OpenWrt platforms: linux/amd64 diff --git a/extra/wolfssl/wolfssl/.github/workflows/haproxy.yml b/extra/wolfssl/wolfssl/.github/workflows/haproxy.yml index 54a52b8c..9c7047bc 100644 --- a/extra/wolfssl/wolfssl/.github/workflows/haproxy.yml +++ b/extra/wolfssl/wolfssl/.github/workflows/haproxy.yml @@ -21,7 +21,7 @@ jobs: install: true - name: Checkout VTest - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: vtest/VTest path: VTest @@ -32,7 +32,7 @@ jobs: run: make FLAGS='-O2 -s -Wall' - name: Checkout HaProxy - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: haproxy/haproxy path: haproxy diff --git a/extra/wolfssl/wolfssl/.github/workflows/hitch.yml b/extra/wolfssl/wolfssl/.github/workflows/hitch.yml index a7f745db..13b933fd 100644 --- a/extra/wolfssl/wolfssl/.github/workflows/hitch.yml +++ b/extra/wolfssl/wolfssl/.github/workflows/hitch.yml @@ -19,11 +19,11 @@ jobs: install: true - name: Upload built lib - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: wolf-install-hitch path: build-dir - retention-days: 1 + retention-days: 5 hitch_check: strategy: @@ -41,13 +41,13 @@ jobs: needs: build_wolfssl steps: - name: Download lib - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: wolf-install-hitch path: build-dir - name: Checkout OSP - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: wolfssl/osp path: osp @@ -59,7 +59,7 @@ jobs: sudo apt-get install -y libev-dev libssl-dev automake python3-docutils flex bison pkg-config make - name: Checkout hitch - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: varnish/hitch ref: 1.7.3 diff --git a/extra/wolfssl/wolfssl/.github/workflows/hostap.yml b/extra/wolfssl/wolfssl/.github/workflows/hostap.yml index 84ea1009..187c735b 100644 --- a/extra/wolfssl/wolfssl/.github/workflows/hostap.yml +++ b/extra/wolfssl/wolfssl/.github/workflows/hostap.yml @@ -40,11 +40,11 @@ jobs: install: true - name: Upload built lib - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ matrix.build_id }} path: build-dir - retention-days: 1 + retention-days: 5 # Build wpa_supplicant with wolfSSL and hostapd with OpenSSL and interop. hostap_test: @@ -113,7 +113,7 @@ jobs: echo Our job run ID is $SHA_SUM - name: Checkout wolfSSL - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: wolfssl @@ -140,7 +140,7 @@ jobs: echo "hostap_debug_flags=-d" >> $GITHUB_ENV - name: Download lib - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: ${{ matrix.config.build_id }} path: build-dir @@ -170,7 +170,7 @@ jobs: sudo rmmod mac80211_hwsim - name: Checkout hostap - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: julek-wolfssl/hostap-mirror path: hostap @@ -185,7 +185,7 @@ jobs: - if: ${{ matrix.config.osp_ref }} name: Checkout OSP - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: wolfssl/osp path: osp @@ -275,7 +275,7 @@ jobs: - name: Upload failure logs if: ${{ failure() && steps.testing.outcome == 'failure' }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: hostap-logs-${{ env.our_job_run_id }} path: hostap/tests/hwsim/logs.zip diff --git a/extra/wolfssl/wolfssl/.github/workflows/krb5.yml b/extra/wolfssl/wolfssl/.github/workflows/krb5.yml index f03237c8..0022d64d 100644 --- a/extra/wolfssl/wolfssl/.github/workflows/krb5.yml +++ b/extra/wolfssl/wolfssl/.github/workflows/krb5.yml @@ -11,19 +11,23 @@ jobs: # This should be a safe limit for the tests to run. timeout-minutes: 5 steps: + - name: workaround high-entropy ASLR + # not needed after either an update to llvm or runner is done + run: sudo sysctl vm.mmap_rnd_bits=28 + - name: Build wolfSSL uses: wolfSSL/actions-build-autotools-project@v1 with: path: wolfssl - configure: --enable-krb CFLAGS='-fsanitize=address' + configure: --enable-krb CC='gcc -fsanitize=address' install: true - name: Upload built lib - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: wolf-install-krb5 path: build-dir - retention-days: 1 + retention-days: 5 krb5_check: strategy: @@ -38,19 +42,19 @@ jobs: needs: build_wolfssl steps: - name: Download lib - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: wolf-install-krb5 path: build-dir - name: Checkout OSP - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: wolfssl/osp path: osp - name: Checkout krb5 - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: krb5/krb5 ref: krb5-${{ matrix.ref }}-final @@ -61,6 +65,10 @@ jobs: run: | patch -p1 < $GITHUB_WORKSPACE/osp/krb5/Patch-for-Kerberos-5-${{ matrix.ref }}.patch + - name: workaround high-entropy ASLR + # not needed after either an update to llvm or runner is done + run: sudo sysctl vm.mmap_rnd_bits=28 + - name: Build krb5 working-directory: ./krb5/src run: | diff --git a/extra/wolfssl/wolfssl/.github/workflows/libssh2.yml b/extra/wolfssl/wolfssl/.github/workflows/libssh2.yml new file mode 100644 index 00000000..dd9d50e6 --- /dev/null +++ b/extra/wolfssl/wolfssl/.github/workflows/libssh2.yml @@ -0,0 +1,58 @@ +name: libssh2 Tests + +on: + workflow_call: + +jobs: + build_wolfssl: + name: Build wolfSSL + # Just to keep it the same as the testing target + runs-on: ubuntu-latest + # This should be a safe limit for the tests to run. + timeout-minutes: 4 + steps: + - name: Build wolfSSL + uses: wolfSSL/actions-build-autotools-project@v1 + with: + path: wolfssl + configure: --enable-all + check: false # config is already tested in many other PRB's + install: true + + - name: Upload built lib + uses: actions/upload-artifact@v4 + with: + name: wolf-install-libssh2 + path: build-dir + retention-days: 5 + + libssh2_check: + strategy: + fail-fast: false + matrix: + # List of releases to test + ref: [ 1.11.0 ] + name: ${{ matrix.ref }} + runs-on: ubuntu-latest + # This should be a safe limit for the tests to run. + timeout-minutes: 8 + needs: build_wolfssl + steps: + - name: Download lib + uses: actions/download-artifact@v4 + with: + name: wolf-install-libssh2 + path: build-dir + + - name: Build and test libssh2 + uses: wolfSSL/actions-build-autotools-project@v1 + with: + repository: libssh2/libssh2 + ref: libssh2-${{ matrix.ref }} + path: libssh2 + configure: --with-crypto=wolfssl --with-libwolfssl-prefix=$GITHUB_WORKSPACE/build-dir + check: true + + - name: Confirm libssh2 built with wolfSSL + working-directory: ./libssh2 + run: ldd src/.libs/libssh2.so | grep wolfssl diff --git a/extra/wolfssl/wolfssl/.github/workflows/main.yml b/extra/wolfssl/wolfssl/.github/workflows/main.yml index b1e63a32..4579eaab 100644 --- a/extra/wolfssl/wolfssl/.github/workflows/main.yml +++ b/extra/wolfssl/wolfssl/.github/workflows/main.yml @@ -43,5 +43,10 @@ jobs: memcached: uses: ./.github/workflows/memcached.yml # TODO: Currently this test fails. Enable it once it becomes passing. +# libssh2: +# uses: ./.github/workflows/libssh2.yml + openssh: + uses: ./.github/workflows/openssh.yml +# TODO: Currently this test fails. Enable it once it becomes passing. # haproxy: # uses: ./.github/workflows/haproxy.yml diff --git a/extra/wolfssl/wolfssl/.github/workflows/memcached.yml b/extra/wolfssl/wolfssl/.github/workflows/memcached.yml index 9a4c8131..fde37018 100644 --- a/extra/wolfssl/wolfssl/.github/workflows/memcached.yml +++ b/extra/wolfssl/wolfssl/.github/workflows/memcached.yml @@ -20,11 +20,11 @@ jobs: run: cp wolfssl/.github/workflows/memcached.sh build-dir/bin - name: Upload built lib - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: wolf-install-memcached path: build-dir - retention-days: 1 + retention-days: 5 memcached_check: strategy: @@ -38,13 +38,13 @@ jobs: needs: build_wolfssl steps: - name: Download lib - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: wolf-install-memcached path: build-dir - name: Checkout OSP - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: wolfssl/osp path: osp @@ -56,7 +56,7 @@ jobs: sudo apt-get install -y libevent-dev libevent-2.1-7 automake pkg-config make libio-socket-ssl-perl - name: Checkout memcached - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: memcached/memcached ref: 1.6.22 diff --git a/extra/wolfssl/wolfssl/.github/workflows/multi-arch.yml b/extra/wolfssl/wolfssl/.github/workflows/multi-arch.yml index e5b9859a..031ca802 100644 --- a/extra/wolfssl/wolfssl/.github/workflows/multi-arch.yml +++ b/extra/wolfssl/wolfssl/.github/workflows/multi-arch.yml @@ -35,7 +35,7 @@ jobs: run: | sudo apt update sudo apt install -y crossbuild-essential-${{ matrix.ARCH }} qemu-user - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build env: CC: ${{ matrix.CC }} diff --git a/extra/wolfssl/wolfssl/.github/workflows/multi-compiler.yml b/extra/wolfssl/wolfssl/.github/workflows/multi-compiler.yml index 48512df2..b63fd0f2 100644 --- a/extra/wolfssl/wolfssl/.github/workflows/multi-compiler.yml +++ b/extra/wolfssl/wolfssl/.github/workflows/multi-compiler.yml @@ -41,7 +41,7 @@ jobs: # This should be a safe limit for the tests to run. timeout-minutes: 4 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build env: CC: ${{ matrix.CC }} diff --git a/extra/wolfssl/wolfssl/.github/workflows/nginx.yml b/extra/wolfssl/wolfssl/.github/workflows/nginx.yml index 05f2ed7c..9f9f81ec 100644 --- a/extra/wolfssl/wolfssl/.github/workflows/nginx.yml +++ b/extra/wolfssl/wolfssl/.github/workflows/nginx.yml @@ -25,11 +25,11 @@ jobs: install: true - name: Upload built lib - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: wolf-install-nginx path: build-dir - retention-days: 1 + retention-days: 5 nginx_check: strategy: @@ -99,7 +99,7 @@ jobs: needs: build_wolfssl steps: - name: Download lib - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: wolf-install-nginx path: build-dir @@ -109,13 +109,13 @@ jobs: sudo cpan -iT Proc::Find Net::SSLeay IO::Socket::SSL - name: Checkout wolfssl-nginx - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: wolfssl/wolfssl-nginx path: wolfssl-nginx - name: Checkout nginx - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: nginx/nginx path: nginx @@ -131,7 +131,7 @@ jobs: run: patch -p1 < ../wolfssl-nginx/nginx-${{ matrix.ref }}-wolfssl-debug.patch - name: Checkout nginx-tests - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: nginx/nginx-tests path: nginx-tests @@ -174,6 +174,10 @@ jobs: run: | echo "nginx_c_flags=-O0" >> $GITHUB_ENV + - name: workaround high-entropy ASLR + # not needed after either an update to llvm or runner is done + run: sudo sysctl vm.mmap_rnd_bits=28 + - name: Build nginx with sanitizer working-directory: nginx run: | @@ -203,4 +207,4 @@ jobs: LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/build-dir/lib \ TMPDIR=$GITHUB_WORKSPACE TEST_NGINX_BINARY=../nginx/objs/nginx \ prove ${{ matrix.sanitize-ok }} - \ No newline at end of file + diff --git a/extra/wolfssl/wolfssl/.github/workflows/openssh.yml b/extra/wolfssl/wolfssl/.github/workflows/openssh.yml new file mode 100644 index 00000000..02838773 --- /dev/null +++ b/extra/wolfssl/wolfssl/.github/workflows/openssh.yml @@ -0,0 +1,68 @@ +name: openssh Tests + +on: + workflow_call: + +jobs: + build_wolfssl: + name: Build wolfSSL + # Just to keep it the same as the testing target + runs-on: ubuntu-latest + # This should be a safe limit for the tests to run. + timeout-minutes: 4 + steps: + - name: Build wolfSSL + uses: wolfSSL/actions-build-autotools-project@v1 + with: + path: wolfssl + configure: >- + --enable-openssh --enable-dsa --with-max-rsa-bits=8192 + --enable-intelasm --enable-sp-asm + install: true + + - name: Upload built lib + uses: actions/upload-artifact@v4 + with: + name: wolf-install-openssh + path: build-dir + retention-days: 5 + + openssh_check: + strategy: + fail-fast: false + matrix: + include: + - git_ref: 'V_9_6_P1' + osp_ver: '9.6' + name: ${{ matrix.ref }} + runs-on: ubuntu-latest + needs: build_wolfssl + steps: + - name: Download lib + uses: actions/download-artifact@v4 + with: + name: wolf-install-openssh + path: build-dir + + - name: Checkout OSP + uses: actions/checkout@v4 + with: + repository: wolfssl/osp + path: osp + + - name: Build and test openssh + uses: wolfSSL/actions-build-autotools-project@v1 + with: + repository: openssh/openssh-portable + ref: ${{ matrix.git_ref }} + path: openssh + patch-file: $GITHUB_WORKSPACE/osp/openssh-patches/openssh-${{ matrix.osp_ver }}.patch + configure: --with-wolfssl=$GITHUB_WORKSPACE/build-dir --with-rpath=-Wl,-rpath= + check: false + + # make tests take >20 minutes. Consider limiting? + - name: Run tests + working-directory: ./openssh + run: | + # Run all the tests except (t-exec) as it takes too long + make file-tests interop-tests extra-tests unit diff --git a/extra/wolfssl/wolfssl/.github/workflows/openvpn.yml b/extra/wolfssl/wolfssl/.github/workflows/openvpn.yml index 10f206ff..314f9a4e 100644 --- a/extra/wolfssl/wolfssl/.github/workflows/openvpn.yml +++ b/extra/wolfssl/wolfssl/.github/workflows/openvpn.yml @@ -19,18 +19,18 @@ jobs: install: true - name: Upload built lib - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: wolf-install-openvpn path: build-dir - retention-days: 1 + retention-days: 5 openvpn_check: strategy: fail-fast: false matrix: # List of refs to test - ref: [ master, release/2.6, v2.6.0 ] + ref: [ release/2.6, v2.6.0, master ] name: ${{ matrix.ref }} runs-on: ubuntu-latest # This should be a safe limit for the tests to run. @@ -38,7 +38,7 @@ jobs: needs: build_wolfssl steps: - name: Download lib - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: wolf-install-openvpn path: build-dir @@ -50,10 +50,14 @@ jobs: linux-libc-dev man2html libcmocka-dev python3-docutils \ libtool automake autoconf libnl-genl-3-dev libnl-genl-3-200 + - name: workaround high-entropy ASLR + # not needed after either an update to llvm or runner is done + run: sudo sysctl vm.mmap_rnd_bits=28 + - if: ${{ matrix.ref != 'master' }} name: Build and test openvpn with fsanitize run: | - echo 'extra_c_flags=CFLAGS="-fsanitize=address -fno-omit-frame-pointer -O2"' >> $GITHUB_ENV + echo 'extra_c_flags=CC="gcc -fsanitize=address" CFLAGS="-fno-omit-frame-pointer -O2"' >> $GITHUB_ENV - name: Build and test openvpn uses: wolfSSL/actions-build-autotools-project@v1 diff --git a/extra/wolfssl/wolfssl/.github/workflows/os-check.yml b/extra/wolfssl/wolfssl/.github/workflows/os-check.yml index 08134c4a..9846efaa 100644 --- a/extra/wolfssl/wolfssl/.github/workflows/os-check.yml +++ b/extra/wolfssl/wolfssl/.github/workflows/os-check.yml @@ -67,6 +67,7 @@ jobs: 'examples/configs/user_settings_min_ecc.h', 'examples/configs/user_settings_wolfboot_keytools.h', 'examples/configs/user_settings_wolftpm.h', + 'examples/configs/user_settings_tls12.h', ] name: make user_setting.h (testwolfcrypt only) runs-on: ${{ matrix.os }} @@ -94,7 +95,7 @@ jobs: # This should be a safe limit for the tests to run. timeout-minutes: 14 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - if: ${{ matrix.os == 'macos-latest' }} run: brew install automake libtool - run: ./autogen.sh @@ -121,10 +122,10 @@ jobs: BUILD_CONFIGURATION: Release BUILD_PLATFORM: x64 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Add MSBuild to PATH - uses: microsoft/setup-msbuild@v1 + uses: microsoft/setup-msbuild@v2 - name: Restore NuGet packages working-directory: ${{env.GITHUB_WORKSPACE}} diff --git a/extra/wolfssl/wolfssl/.github/workflows/packaging.yml b/extra/wolfssl/wolfssl/.github/workflows/packaging.yml index 42e21359..50f2a086 100644 --- a/extra/wolfssl/wolfssl/.github/workflows/packaging.yml +++ b/extra/wolfssl/wolfssl/.github/workflows/packaging.yml @@ -11,7 +11,7 @@ jobs: timeout-minutes: 10 steps: - name: Checkout wolfSSL - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Configure wolfSSL run: | diff --git a/extra/wolfssl/wolfssl/.github/workflows/stunnel.yml b/extra/wolfssl/wolfssl/.github/workflows/stunnel.yml index ac25126d..a93a0477 100644 --- a/extra/wolfssl/wolfssl/.github/workflows/stunnel.yml +++ b/extra/wolfssl/wolfssl/.github/workflows/stunnel.yml @@ -19,11 +19,11 @@ jobs: install: true - name: Upload built lib - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: wolf-install-stunnel path: build-dir - retention-days: 1 + retention-days: 5 stunnel_check: strategy: @@ -38,13 +38,13 @@ jobs: needs: build_wolfssl steps: - name: Download lib - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: wolf-install-stunnel path: build-dir - name: Checkout OSP - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: wolfssl/osp path: osp diff --git a/extra/wolfssl/wolfssl/.github/workflows/zephyr.yml b/extra/wolfssl/wolfssl/.github/workflows/zephyr.yml index decdba25..c7f1bc8e 100644 --- a/extra/wolfssl/wolfssl/.github/workflows/zephyr.yml +++ b/extra/wolfssl/wolfssl/.github/workflows/zephyr.yml @@ -64,10 +64,10 @@ jobs: - name: Install zephyr SDK run: | - wget -q https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v${{ matrix.config.zephyr-sdk }}/zephyr-sdk-${{ matrix.config.zephyr-sdk }}_linux-x86_64.tar.xz - tar xf zephyr-sdk-${{ matrix.config.zephyr-sdk }}_linux-x86_64.tar.xz + wget -q https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v${{ matrix.config.zephyr-sdk }}/zephyr-sdk-${{ matrix.config.zephyr-sdk }}_linux-x86_64_minimal.tar.xz + tar xf zephyr-sdk-${{ matrix.config.zephyr-sdk }}_linux-x86_64_minimal.tar.xz cd zephyr-sdk-${{ matrix.config.zephyr-sdk }} - ./setup.sh -h -c + ./setup.sh -h -c -t x86_64-zephyr-elf - name: Run wolfssl test id: wolfssl-test @@ -97,7 +97,7 @@ jobs: - name: Upload failure logs if: ${{ failure() && (steps.wolfssl-test.outcome == 'failure' || steps.wolfssl-tls-sock.outcome == 'failure' || steps.wolfssl-tls-thread.outcome == 'failure') }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: zephyr-client-test-logs path: logs.zip diff --git a/extra/wolfssl/wolfssl/.gitignore b/extra/wolfssl/wolfssl/.gitignore index 5adfbf31..5bd31033 100644 --- a/extra/wolfssl/wolfssl/.gitignore +++ b/extra/wolfssl/wolfssl/.gitignore @@ -82,16 +82,19 @@ snifftest output mcapi/test testsuite/testsuite -tests/unit testsuite/testsuite.test +testsuite/*.der +testsuite/*.pem +testsuite/*.raw +testsuite/*.obj +testsuite/*.pdb +testsuite/*.idb +tests/unit tests/unit.test tests/bio_write_test.txt tests/test-log-dump-to-file.txt tests/cert_cache.tmp test-write-dhparams.pem -testsuite/*.der -testsuite/*.pem -testsuite/*.raw cert.der cert.pem certecc.der @@ -286,23 +289,6 @@ mqx/wolfcrypt_benchmark/.settings mqx/wolfcrypt_benchmark/.cwGeneratedFileSetLog mqx/wolfcrypt_benchmark/SaAnalysispointsManager.apconfig -# User Crypto example build -wolfcrypt/user-crypto/aclocal.m4 -wolfcrypt/user-crypto/config.guess -wolfcrypt/user-crypto/autom4te.cache -wolfcrypt/user-crypto/config.log -wolfcrypt/user-crypto/config.status -wolfcrypt/user-crypto/config.sub -wolfcrypt/user-crypto/depcomp -wolfcrypt/user-crypto/install-sh -wolfcrypt/user-crypto/libtool -wolfcrypt/user-crypto/ltmain.sh -wolfcrypt/user-crypto/m4 -wolfcrypt/user-crypto/missing -wolfcrypt/user-crypto/Makefile.in -wolfcrypt/user-crypto/lib/libusercrypto.* -*.hzs - # wolfSSL CSharp wrapper wrapper/CSharp/x64/ @@ -339,6 +325,10 @@ wolfcrypt/src/port/intel/qat_test # Arduino Generated Files /IDE/ARDUINO/wolfSSL scripts/memtest.txt +/IDE/ARDUINO/Arduino_README_prepend.md.tmp +/IDE/ARDUINO/library.properties.tmp +/IDE/ARDUINO/library.properties.tmp.backup +/IDE/ARDUINO/PREPENDED_README.md # Doxygen generated files doc/doxygen_warnings @@ -415,7 +405,7 @@ XXX-fips-test # Generated user_settings_asm.h. user_settings_asm.h -# VisualGD +# VisualGDB **/.visualgdb # Espressif sdk config default should be saved in sdkconfig.defaults @@ -423,6 +413,12 @@ user_settings_asm.h /IDE/Espressif/**/sdkconfig /IDE/Espressif/**/sdkconfig.old +# MPLAB +/IDE/MPLABX16/wolfssl.X/dist/default/ +/IDE/MPLABX16/wolfssl.X/.generated_files +/IDE/MPLABX16/wolfcrypt_test.X/dist/default/ +/IDE/MPLABX16/wolfcrypt_test.X/.generated_files + # auto-created CMake backups **/CMakeLists.txt.old diff --git a/extra/wolfssl/wolfssl/CMakeLists.txt b/extra/wolfssl/wolfssl/CMakeLists.txt index 337b0d61..bf6bc752 100644 --- a/extra/wolfssl/wolfssl/CMakeLists.txt +++ b/extra/wolfssl/wolfssl/CMakeLists.txt @@ -28,7 +28,7 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_BINARY_DIR}") You must delete them, or cmake will refuse to work.") endif() -project(wolfssl VERSION 5.6.6 LANGUAGES C ASM) +project(wolfssl VERSION 5.7.0 LANGUAGES C ASM) # Set WOLFSSL_ROOT if not already defined if ("${WOLFSSL_ROOT}" STREQUAL "") @@ -42,16 +42,19 @@ else() endif() # shared library versioning -# increment if interfaces have been added, removed or changed -set(LIBTOOL_CURRENT 42) -# increment if source code has changed set to zero if current is incremented -set(LIBTOOL_REVISION 0) -# increment if interfaces have been added set to zero if interfaces have been -# removed or changed -set(LIBTOOL_AGE 0) +# increment if interfaces have been removed or changed +set(WOLFSSL_LIBRARY_VERSION_FIRST 42) -math(EXPR LIBTOOL_SO_VERSION "${LIBTOOL_CURRENT} - ${LIBTOOL_AGE}") -set(LIBTOOL_FULL_VERSION ${LIBTOOL_SO_VERSION}.${LIBTOOL_AGE}.${LIBTOOL_REVISION}) +# increment if interfaces have been added +# set to zero if WOLFSSL_LIBRARY_VERSION_FIRST is incremented +set(WOLFSSL_LIBRARY_VERSION_SECOND 1) + +# increment if source code has changed +# set to zero if WOLFSSL_LIBRARY_VERSION_FIRST is incremented or +# WOLFSSL_LIBRARY_VERSION_SECOND is incremented +set(WOLFSSL_LIBRARY_VERSION_THIRD 0) + +set(LIBTOOL_FULL_VERSION ${WOLFSSL_LIBRARY_VERSION_FIRST}.${WOLFSSL_LIBRARY_VERSION_SECOND}.${WOLFSSL_LIBRARY_VERSION_THIRD}) set(WOLFSSL_DEFINITIONS) set(WOLFSSL_LINK_LIBS) @@ -130,9 +133,15 @@ if(HAVE___UINT128_T) list(APPEND WOLFSSL_DEFINITIONS "-DHAVE___UINT128_T") endif() -include(TestBigEndian) - -test_big_endian(WORDS_BIGENDIAN) +if(CMAKE_VERSION VERSION_LESS "3.20") + # TestBigEndian was deprecated in 3.20 + include(TestBigEndian) + test_big_endian(IS_BIG_ENDIAN) + set(CMAKE_C_BYTE_ORDER "LITTLE_ENDIAN") + if(IS_BIG_ENDIAN) + set(CMAKE_C_BYTE_ORDER "BIG_ENDIAN") + endif() +endif() # Thread local storage include(CheckCSourceCompiles) @@ -215,6 +224,8 @@ if(WOLFSSL_REPRODUCIBLE_BUILD) set(CMAKE_C_ARCHIVE_FINISH " -D ") endif() +add_option("WOLFSSL_INSTALL" "Create install target for WolfSSL project" "yes" "yes;no") + # Support for forcing 32-bit mode # TODO: detect platform from other options add_option("WOLFSSL_32BIT" @@ -574,7 +585,7 @@ if(WOLFSSL_LEAN_PSK OR (WOLFSSL_LEAN_TLS AND NOT WOLFSSL_TLS13)) override_cache(WOLFSSL_AESGCM "no") endif() -if(WOLFSSL_AESGCM AND NOT WORDS_BIGENDIAN) +if(WOLFSSL_AESGCM AND CMAKE_C_BYTE_ORDER STREQUAL "LITTLE_ENDIAN") override_cache(WOLFSSL_AESGCM "4bit") endif() @@ -1662,17 +1673,14 @@ if(WOLFSSL_FAST_MATH) list(APPEND WOLFSSL_DEFINITIONS "-DUSE_FAST_MATH") set(WOLFSSL_SLOWMATH "no") endif() - - if("${CMAKE_SYSTEM_PROCESSOR}" MATCHES "x86_64|AMD64") - # Have settings.h set FP_MAX_BITS higher if user didn't set directly - list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_X86_64_BUILD") - endif() endif() # TODO: - Fast huge math # Set processor-specific build macros if("${CMAKE_SYSTEM_PROCESSOR}" MATCHES "x86_64|AMD64") + set(WOLFSSL_X86_64_BUILD ON) + add_option("WOLFSSL_X86_64_BUILD_ASM" "Build ASM files" "yes" "yes;no") list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_X86_64_BUILD") elseif("${CMAKE_SYSTEM_PROCESSOR}" MATCHES "aarch64|arm64") list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_AARCH64_BUILD") @@ -1753,13 +1761,15 @@ else() list(APPEND WOLFSSL_DEFINITIONS "-DWC_NO_ASYNC_THREADING") endif() -# TODO: - cryptodev -# - Session export +# TODO: - Session export add_option("WOLFSSL_CRYPTOCB" "Enable crypto callbacks (default: disabled)" "no" "yes;no") +add_option("WOLFSSL_PKCALLBACKS" + "Enable public key callbacks (default: disabled)" + "no" "yes;no") add_option("WOLFSSL_OLD_NAMES" "Keep backwards compat with old names (default: enabled)" @@ -1960,6 +1970,11 @@ if(WOLFSSL_CRYPTOCB) list(APPEND WOLFSSL_DEFINITIONS "-DWOLF_CRYPTO_CB") endif() +# Public Key Callbacks +if(WOLFSSL_PKCALLBACKS) + list(APPEND WOLFSSL_DEFINITIONS "-DHAVE_PK_CALLBACKS") +endif() + if(WOLFSSL_OCSPSTAPLING) list(APPEND WOLFSSL_DEFINITIONS "-DHAVE_CERTIFICATE_STATUS_REQUEST" "-DHAVE_TLS_EXTENSIONS") override_cache(WOLFSSL_OCSP "yes") @@ -2075,7 +2090,7 @@ endif() # Suppress some warnings about separate compilation, inlining add_definitions("-DWOLFSSL_IGNORE_FILE_WARN") # Generate user options header -message("Generating user options header...") +message(STATUS "Generating user options header...") if (${CMAKE_DISABLE_SOURCE_CHANGES}) set(WOLFSSL_BUILD_OUT_OF_TREE_DEFAULT "${CMAKE_DISABLE_SOURCE_CHANGES}") else() @@ -2198,7 +2213,7 @@ endif() set_target_properties(wolfssl PROPERTIES - SOVERSION ${LIBTOOL_SO_VERSION} + SOVERSION ${WOLFSSL_LIBRARY_VERSION_FIRST} VERSION ${LIBTOOL_FULL_VERSION} ) @@ -2516,82 +2531,96 @@ list(JOIN HEADER_EXCLUDE "|" EXCLUDED_HEADERS_REGEX) string(PREPEND EXCLUDED_HEADERS_REGEX "(") string(APPEND EXCLUDED_HEADERS_REGEX ")") -set(INSTALLED_EXAMPLES - ${CMAKE_CURRENT_SOURCE_DIR}/examples/echoserver/echoserver.c - ${CMAKE_CURRENT_SOURCE_DIR}/examples/sctp/sctp-server.c - ${CMAKE_CURRENT_SOURCE_DIR}/examples/sctp/sctp-client-dtls.c - ${CMAKE_CURRENT_SOURCE_DIR}/examples/sctp/sctp-client.c - ${CMAKE_CURRENT_SOURCE_DIR}/examples/sctp/sctp-server-dtls.c - ${CMAKE_CURRENT_SOURCE_DIR}/examples/echoclient/echoclient.c - ${CMAKE_CURRENT_SOURCE_DIR}/examples/server/server.c - ${CMAKE_CURRENT_SOURCE_DIR}/examples/benchmark/tls_bench.c - ${CMAKE_CURRENT_SOURCE_DIR}/examples/client/client.c) - -# Install the library -install(TARGETS wolfssl - EXPORT wolfssl-targets - LIBRARY DESTINATION lib - ARCHIVE DESTINATION lib - RUNTIME DESTINATION bin - ) -# Install the headers -install(DIRECTORY ${WOLFSSL_OUTPUT_BASE}/wolfssl/ - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/wolfssl - FILES_MATCHING PATTERN "*.h" - REGEX ${EXCLUDED_HEADERS_REGEX} EXCLUDE) -install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/wolfssl/ - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/wolfssl - FILES_MATCHING PATTERN "*.h" - REGEX ${EXCLUDED_HEADERS_REGEX} EXCLUDE) - -# Install the examples -install(FILES ${INSTALLED_EXAMPLES} - DESTINATION ${CMAKE_INSTALL_DOCDIR}/example) -# Install README.txt and taoCert.txt -install(FILES - ${CMAKE_CURRENT_SOURCE_DIR}/doc/README.txt - ${CMAKE_CURRENT_SOURCE_DIR}/certs/taoCert.txt - DESTINATION ${CMAKE_INSTALL_DOCDIR}) -# Install the export set -install(EXPORT wolfssl-targets - DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/wolfssl - FILE wolfssl-targets.cmake - NAMESPACE wolfssl::) - -# TODO: Distro build + rules for what to include in the distro. -# See various include.am files. - -set(prefix ${CMAKE_INSTALL_PREFIX}) -set(exec_prefix "\${prefix}") -set(libdir "\${exec_prefix}/lib") -set(includedir "\${prefix}/include") -set(VERSION ${PROJECT_VERSION}) - -configure_file(support/wolfssl.pc.in ${CMAKE_CURRENT_BINARY_DIR}/support/wolfssl.pc @ONLY) -install(FILES ${CMAKE_CURRENT_BINARY_DIR}/support/wolfssl.pc - DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) - -include(CMakePackageConfigHelpers) -configure_package_config_file(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Config.cmake.in - "${CMAKE_CURRENT_BINARY_DIR}/wolfssl-config.cmake" - INSTALL_DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/wolfssl" - NO_SET_AND_CHECK_MACRO - NO_CHECK_REQUIRED_COMPONENTS_MACRO -) +if(WOLFSSL_INSTALL) + + set(INSTALLED_EXAMPLES + ${CMAKE_CURRENT_SOURCE_DIR}/examples/echoserver/echoserver.c + ${CMAKE_CURRENT_SOURCE_DIR}/examples/sctp/sctp-server.c + ${CMAKE_CURRENT_SOURCE_DIR}/examples/sctp/sctp-client-dtls.c + ${CMAKE_CURRENT_SOURCE_DIR}/examples/sctp/sctp-client.c + ${CMAKE_CURRENT_SOURCE_DIR}/examples/sctp/sctp-server-dtls.c + ${CMAKE_CURRENT_SOURCE_DIR}/examples/echoclient/echoclient.c + ${CMAKE_CURRENT_SOURCE_DIR}/examples/server/server.c + ${CMAKE_CURRENT_SOURCE_DIR}/examples/benchmark/tls_bench.c + ${CMAKE_CURRENT_SOURCE_DIR}/examples/client/client.c) -export(EXPORT wolfssl-targets - FILE "${CMAKE_CURRENT_BINARY_DIR}/wolfssl-targets.cmake" - NAMESPACE wolfssl:: -) -write_basic_package_version_file( - "${CMAKE_CURRENT_BINARY_DIR}/wolfssl-config-version.cmake" - VERSION "${wolfssl_VERSION_MAJOR}.${wolfssl_VERSION_MINOR}" - COMPATIBILITY AnyNewerVersion -) + # Install the library + install(TARGETS wolfssl + EXPORT wolfssl-targets + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib + RUNTIME DESTINATION bin + ) + # Install the headers + install(DIRECTORY ${WOLFSSL_OUTPUT_BASE}/wolfssl/ + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/wolfssl + FILES_MATCHING PATTERN "*.h" + REGEX ${EXCLUDED_HEADERS_REGEX} EXCLUDE) + install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/wolfssl/ + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/wolfssl + FILES_MATCHING PATTERN "*.h" + REGEX ${EXCLUDED_HEADERS_REGEX} EXCLUDE) + + # Install the examples + install(FILES ${INSTALLED_EXAMPLES} + DESTINATION ${CMAKE_INSTALL_DOCDIR}/example) + # Install README.txt and taoCert.txt + install(FILES + ${CMAKE_CURRENT_SOURCE_DIR}/doc/README.txt + ${CMAKE_CURRENT_SOURCE_DIR}/certs/taoCert.txt + DESTINATION ${CMAKE_INSTALL_DOCDIR}) + # Install the export set + install(EXPORT wolfssl-targets + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/wolfssl + FILE wolfssl-targets.cmake + NAMESPACE wolfssl::) + + # TODO: Distro build + rules for what to include in the distro. + # See various include.am files. + + set(prefix ${CMAKE_INSTALL_PREFIX}) + set(exec_prefix "\${prefix}") + set(libdir "\${exec_prefix}/lib") + set(includedir "\${prefix}/include") + set(VERSION ${PROJECT_VERSION}) + + # Setting libm in Libs.private of wolfssl.pc. + # See "Link Libraries" in above about `m` insertion to LINK_LIBRARIES + get_target_property(_wolfssl_dep_libs wolfssl LINK_LIBRARIES) + list(FIND _wolfssl_dep_libs m _dep_libm) + if ("${_dep_libm}" GREATER -1) + set(LIBM -lm) + else() + set(LIBM) + endif() -install(FILES - ${CMAKE_CURRENT_BINARY_DIR}/wolfssl-config.cmake - ${CMAKE_CURRENT_BINARY_DIR}/wolfssl-config-version.cmake - DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/wolfssl -) + configure_file(support/wolfssl.pc.in ${CMAKE_CURRENT_BINARY_DIR}/support/wolfssl.pc @ONLY) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/support/wolfssl.pc + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) + + include(CMakePackageConfigHelpers) + configure_package_config_file(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Config.cmake.in + "${CMAKE_CURRENT_BINARY_DIR}/wolfssl-config.cmake" + INSTALL_DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/wolfssl" + NO_SET_AND_CHECK_MACRO + NO_CHECK_REQUIRED_COMPONENTS_MACRO + ) + + export(EXPORT wolfssl-targets + FILE "${CMAKE_CURRENT_BINARY_DIR}/wolfssl-targets.cmake" + NAMESPACE wolfssl:: + ) + + write_basic_package_version_file( + "${CMAKE_CURRENT_BINARY_DIR}/wolfssl-config-version.cmake" + VERSION "${wolfssl_VERSION_MAJOR}.${wolfssl_VERSION_MINOR}" + COMPATIBILITY AnyNewerVersion + ) + + install(FILES + ${CMAKE_CURRENT_BINARY_DIR}/wolfssl-config.cmake + ${CMAKE_CURRENT_BINARY_DIR}/wolfssl-config-version.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/wolfssl + ) +endif() diff --git a/extra/wolfssl/wolfssl/ChangeLog.md b/extra/wolfssl/wolfssl/ChangeLog.md index 586adaa4..c3b24757 100644 --- a/extra/wolfssl/wolfssl/ChangeLog.md +++ b/extra/wolfssl/wolfssl/ChangeLog.md @@ -1,3 +1,97 @@ +# wolfSSL Release 5.7.0 (Mar 20, 2024) + +Release 5.7.0 has been developed according to wolfSSL's development and QA +process (see link below) and successfully passed the quality criteria. +https://www.wolfssl.com/about/wolfssl-software-development-process-quality-assurance + +NOTE: * --enable-heapmath is being deprecated and will be removed by end of 2024 + +NOTE: In future releases, --enable-des3 (which is disabled by default) will be insufficient in itself to enable DES3 in TLS cipher suites. A new option, --enable-des3-tls-suites, will need to be supplied in addition. This option should only be used in backward compatibility scenarios, as it is inherently insecure. + +NOTE: This release switches the default ASN.1 parser to the new ASN template code. If the original ASN.1 code is preferred define `WOLFSSL_ASN_ORIGINAL` to use it. See PR #7199. + + +## Vulnerabilities +* [High] CVE-2024-0901 Potential denial of service and out of bounds read. Affects TLS 1.3 on the server side when accepting a connection from a malicious TLS 1.3 client. If using TLS 1.3 on the server side it is recommended to update the version of wolfSSL used. Fixed in this GitHub pull request https://github.com/wolfSSL/wolfssl/pull/7099 + + +* [Med] CVE-2024-1545 Fault Injection vulnerability in RsaPrivateDecryption function that potentially allows an attacker that has access to the same system with a victims process to perform a Rowhammer fault injection. Thanks to Junkai Liang, Zhi Zhang, Xin Zhang, Qingni Shen for the report (Peking University, The University of Western Australia)." +Fixed in this GitHub pull request https://github.com/wolfSSL/wolfssl/pull/7167 + + +* [Med] Fault injection attack with EdDSA signature operations. This affects ed25519 sign operations where the system could be susceptible to Rowhammer attacks. Thanks to Junkai Liang, Zhi Zhang, Xin Zhang, Qingni Shen for the report (Peking University, The University of Western Australia). +Fixed in this GitHub pull request https://github.com/wolfSSL/wolfssl/pull/7212 + + +## New Feature Additions + +* Added --enable-experimental configure flag to gate out features that are currently experimental. Now liboqs, kyber, lms, xmss, and dual-alg-certs require the --enable-experimental flag. + +### POST QUANTUM SUPPORT ADDITIONS +* Experimental framework for using wolfSSL’s XMSS implementation (PR 7161) +* Experimental framework for using wolfSSL’s LMS implementation (PR 7283) +* Experimental wolfSSL Kyber implementation and assembly optimizations, enabled with --enable-experimental --enable-kyber (PR 7318) +* Experimental support for post quantum dual key/signature certificates. A few known issues and sanitizer checks are in progress with this feature. Enabled with the configure flags --enable-experimental --enable-dual-alg-certs (PR 7112) +* CryptoCb support for PQC algorithms (PR 7110) + +### OTHER FEATURE ADDITIONS +* The Linux kernel module now supports registration of AES-GCM, AES-XTS, AES-CBC, and AES-CFB with the kernel cryptosystem through the new --enable-linuxkm-lkcapi-register option, enabling automatic use of wolfCrypt implementations by the dm-crypt/luks and ESP subsystems. In particular, wolfCrypt AES-XTS with –enable-aesni is faster than the native kernel implementation. +* CryptoCb hook to one-shot CMAC functions (PR 7059) +* BER content streaming support for PKCS7_VerifySignedData and sign/encrypt operations (PR 6961 & 7184) +* IoT-Safe SHA-384 and SHA-512 support (PR 7176) +* I/O callbacks for content and output with PKCS7 bundle sign/encrypt to reduce peak memory usage (PR 7272) +* Microchip PIC24 support and example project (PR 7151) +* AutoSAR shim layer for RNG, SHA256, and AES (PR 7296) +* wolfSSL_CertManagerUnloadIntermediateCerts API to clear intermediate certs added to certificate store (PR 7245) +* Implement SSL_get_peer_signature_nid and SSL_get_peer_signature_type_nid (PR 7236) + + +## Enhancements and Optimizations + +* Remove obsolete user-crypto functionality and Intel IPP support (PR 7097) +* Support for RSA-PSS signatures with CRL use (PR 7119) +* Enhancement for AES-GCM use with Xilsecure on Microblaze (PR 7051) +* Support for crypto cb only build with ECC and NXP CAAM (PR 7269) +* Improve liboqs integration adding locking and init/cleanup functions (PR 7026) +* Prevent memory access before clientSession->serverRow and clientSession->serverIdx are sanitized (PR 7096) +* Enhancements to reproducible build (PR 7267) +* Update Arduino example TLS Client/Server and improve support for ESP32 (PR 7304 & 7177) +* XC32 compiler version 4.x compatibility (PR 7128) +* Porting for build on PlayStation 3 and 4 (PR 7072) +* Improvements for Espressif use; SHA HW/SW selection and use on ESP32-C2/ESP8684, wolfSSL_NewThread() type, component cmake fix, and update TLS client example for ESP8266 (PR 7081, 7173, 7077, 7148, 7240) +* Allow crypto callbacks with SHA-1 HW (PR 7087) +* Update OpenSSH port to version 9.6p1(PR 7203) +* ARM Thumb2 enhancements, AES-GCM support for GCM_SMALL, alignment fix on key, fix for ASM clobber list (PR 7291,7301,7221) +* Expand heap hint support for static memory build with more x509 functions (PR 7136) +* Improving ARMv8 ChaCha20 ASM (alignment) (PR 7182) +* Unknown extension callback wolfSSL_CertManagerSetUnknownExtCallback added to CertManager (PR 7194) +* Implement wc_rng_new_ex for use with devID’s with crypto callback (PR 7271) +* Allow reading 0-RTT data after writing 0.5-RTT data (PR 7102) +* Send alert on bad PSK binder error (PR 7235) +* Enhancements to CMake build files for use with cross compiling (PR 7188) + + +## Fixes + +* Fix for checking result of MAC verify when no AAD is used with AES-GCM and Xilinx Xilsecure (PR 7051) +* Fix for Aria sign use (PR 7082) +* Fix for invalid `dh_ffdhe_test` test case using Intel QuickAssist (PR 7085) +* Fixes for TI AES and SHA on TM4C with HW acceleration and add full AES GCM and CCM support with TLS (PR 7018) +* Fixes for STM32 PKA use with ECC (PR 7098) +* Fixes for TLS 1.3 with crypto callbacks to offload KDF / HMAC operation (PR 7070) +* Fix include path for FSP 3.5 on Renesas RA6M4 (PR 7101) +* Siphash x64 asm fix for use with older compilers (PR 7299) +* Fix for SGX build with SP (PR 7308) +* Fix to Make it mandatory that the cookie is sent back in new ClientHello when seen in a HelloRetryRequest with (PR 7190) +* Fix for wrap around behavior with BIO pairs (PR 7169) +* OCSP fixes for parsing of response correctly when there was a revocation reason and returning correct error value with date checks (PR 7241 & 7255) +* Fix build with `NO_STDIO_FILESYSTEM` and improve checks for `XGETENV` (PR 7150) +* Fix for DTLS sequence number and cookie when downgrading DTLS version (PR 7214) +* Fix for write_dup use with chacha-poly cipher suites (PR 7206) +* Fix for multiple handshake messages in one record failing with OUT_OF_ORDER_E when downgrading from TLS 1.3 to TLS 1.2 (PR 7141) +* Fix for AES ECB build with Thumb and alignment (PR 7094) +* Fix for negotiate handshake until the end in wolfSSL_read/wolfSSL_write if hitting an edge case with want read/write (PR 7237) + # wolfSSL Release 5.6.6 (Dec 19, 2023) Release 5.6.6 has been developed according to wolfSSL's development and QA diff --git a/extra/wolfssl/wolfssl/Docker/Dockerfile b/extra/wolfssl/wolfssl/Docker/Dockerfile index 388169e6..b45cff4d 100644 --- a/extra/wolfssl/wolfssl/Docker/Dockerfile +++ b/extra/wolfssl/wolfssl/Docker/Dockerfile @@ -3,10 +3,13 @@ FROM $DOCKER_BASE_IMAGE USER root +# Set timezone to UTC +RUN ln -snf /usr/share/zoneinfo/UTC /etc/localtime && echo UTC > /etc/timezone + ARG DEPS_WOLFSSL="build-essential autoconf libtool clang clang-tools zlib1g-dev libuv1-dev libpam0g-dev valgrind git linux-headers-generic gcc-multilib g++-multilib libpcap-dev bubblewrap gdb iputils-ping lldb bsdmainutils netcat binutils-arm-linux-gnueabi binutils-aarch64-linux-gnu" ARG DEPS_LIBOQS="astyle cmake gcc ninja-build libssl-dev python3-pytest python3-pytest-xdist unzip xsltproc doxygen graphviz python3-yaml valgrind git" ARG DEPS_UDP_PROXY="wget libevent-dev" -ARG DEPS_TESTS="abi-dumper libcurl4-openssl-dev tcpdump" +ARG DEPS_TESTS="abi-dumper libcurl4-openssl-dev tcpdump libpsl-dev python3-pandas python3-tabulate libnl-genl-3-dev libcap-ng-dev" ARG DEPS_TOOLS="ccache" RUN DEBIAN_FRONTEND=noninteractive apt update && apt install -y apt-utils \ && apt install -y ${DEPS_WOLFSSL} ${DEPS_LIBOQS} ${DEPS_UDP_PROXY} ${DEPS_TESTS} ${DEPS_TOOLS} \ diff --git a/extra/wolfssl/wolfssl/Docker/Dockerfile.cross-compiler b/extra/wolfssl/wolfssl/Docker/Dockerfile.cross-compiler index a89a9d5b..a9868d38 100644 --- a/extra/wolfssl/wolfssl/Docker/Dockerfile.cross-compiler +++ b/extra/wolfssl/wolfssl/Docker/Dockerfile.cross-compiler @@ -3,7 +3,7 @@ FROM $DOCKER_BASE_IMAGE USER root -ARG DEPS_TESTING="gcc-arm-linux-gnueabi gcc-aarch64-linux-gnu" +ARG DEPS_TESTING="gcc-arm-linux-gnueabi gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu" RUN DEBIAN_FRONTEND=noninteractive apt update \ && apt install -y ${DEPS_TESTING} \ && apt clean -y && rm -rf /var/lib/apt/lists/* diff --git a/extra/wolfssl/wolfssl/Docker/yocto/Dockerfile b/extra/wolfssl/wolfssl/Docker/yocto/Dockerfile new file mode 100644 index 00000000..5fda38f9 --- /dev/null +++ b/extra/wolfssl/wolfssl/Docker/yocto/Dockerfile @@ -0,0 +1,27 @@ +FROM ubuntu + +# Set timezone to UTC +RUN ln -snf /usr/share/zoneinfo/UTC /etc/localtime && echo UTC > /etc/timezone + +RUN DEBIAN_FRONTEND=noninteractive apt update && apt install -y gawk wget git diffstat unzip texinfo gcc build-essential chrpath socat cpio python3 python3-pip python3-pexpect xz-utils debianutils iputils-ping python3-git python3-jinja2 libegl1-mesa libsdl1.2-dev python3-subunit mesa-common-dev zstd liblz4-tool file locales libacl1 vim && apt clean -y && rm -rf /var/lib/apt/lists/* +RUN locale-gen en_US.UTF-8 + +# Add in non-root user +ENV UID_OF_DOCKERUSER 1000 +RUN useradd -m -s /bin/bash -g users -u ${UID_OF_DOCKERUSER} dockerUser +RUN chown -R dockerUser:users /home/dockerUser && chown dockerUser:users /opt + +USER dockerUser + +RUN cd /opt && git clone git://git.yoctoproject.org/poky +WORKDIR /opt/poky + +ARG YOCTO_VERSION=kirkstone +RUN git checkout -t origin/${YOCTO_VERSION} -b ${YOCTO_VERSION} && git pull + +RUN git clone --single-branch --branch=${YOCTO_VERSION} https://github.com/wolfSSL/meta-wolfssl.git && \ + /bin/bash -c "source oe-init-build-env" && \ + echo 'IMAGE_INSTALL:append = " wolfssl wolfclu wolfssh wolfmqtt wolftpm wolfclu "' >> /opt/poky/build/conf/local.conf && \ + sed -i '/\/opt\/poky\/meta-poky \\/a \\t/opt/poky/meta-wolfssl \\' /opt/poky/build/conf/bblayers.conf + +RUN /bin/bash -c "source oe-init-build-env && bitbake core-image-minimal" diff --git a/extra/wolfssl/wolfssl/Docker/yocto/buildAndPush.sh b/extra/wolfssl/wolfssl/Docker/yocto/buildAndPush.sh new file mode 100755 index 00000000..d76a603e --- /dev/null +++ b/extra/wolfssl/wolfssl/Docker/yocto/buildAndPush.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Assume we're in wolfssl/Docker/yocto +WOLFSSL_DIR=$(builtin cd ${BASH_SOURCE%/*}/../..; pwd) + +DOCKER_BUILD_OPTIONS="$1" +if [ "${DOCKER_BASE_IMAGE}" != "" ]; then + DOCKER_BUILD_OPTIONS+=" --build-arg DOCKER_BASE_IMAGE=${DOCKER_BASE_IMAGE}" +fi + +NUM_FAILURES=0 + +CUR_DATE=$(date -u +%F) +for ver in kirkstone dunfell; do + echo "Building wolfssl/yocto:${ver}-${CUR_DATE} as ${DOCKER_BUILD_OPTIONS}" + docker build -t wolfssl/yocto:${ver}-${CUR_DATE} --build-arg YOCTO_VERSION=${ver} -f Dockerfile "${WOLFSSL_DIR}/Docker/yocto" && \ + docker tag wolfssl/yocto:${ver}-${CUR_DATE} wolfssl/yocto:${ver}-latest + if [ $? -eq 0 ]; then + echo "Pushing containers to DockerHub" + docker push wolfssl/yocto:${ver}-${CUR_DATE} && docker push wolfssl/yocto:${ver}-latest + else + echo "Warning: Build wolfssl/yocto:${ver} failed. Continuing" + ((NUM_FAILURES++)) + fi +done + +echo "Script completed in $SECONDS seconds. Had $NUM_FAILURES failures." diff --git a/extra/wolfssl/wolfssl/IDE/ARDUINO/Arduino_README_prepend.md b/extra/wolfssl/wolfssl/IDE/ARDUINO/Arduino_README_prepend.md new file mode 100644 index 00000000..594a0678 --- /dev/null +++ b/extra/wolfssl/wolfssl/IDE/ARDUINO/Arduino_README_prepend.md @@ -0,0 +1,13 @@ +# Arduino wolfSSL Library + +This library is restructured from [wolfSSL](https://github.com/wolfSSL/wolfssl/) Release ${WOLFSSL_VERSION} for the Arduino platform. + +The Official wolfSSL Arduino Library is found in [The Library Manager index](http://downloads.arduino.cc/libraries/library_index.json). + +See the [Arduino-wolfSSL logs](https://downloads.arduino.cc/libraries/logs/github.com/wolfSSL/Arduino-wolfSSL/). + +## Arduino Releases + +The first Official wolfSSL Arduino Library is `5.6.6-Arduino.1`: a slightly modified, post [release 5.6.6](https://github.com/wolfSSL/wolfssl/releases/tag/v5.6.6-stable) version update. + +See other [wolfSSL releases versions](https://github.com/wolfSSL/wolfssl/releases). The `./wolfssl-arduino.sh INSTALL` [script](https://github.com/wolfSSL/wolfssl/tree/master/IDE/ARDUINO) can be used to install specific GitHub versions as needed. diff --git a/extra/wolfssl/wolfssl/IDE/ARDUINO/README.md b/extra/wolfssl/wolfssl/IDE/ARDUINO/README.md index 4c4e10da..75c25a20 100644 --- a/extra/wolfssl/wolfssl/IDE/ARDUINO/README.md +++ b/extra/wolfssl/wolfssl/IDE/ARDUINO/README.md @@ -1,30 +1,89 @@ -### wolfSSL with Arduino +# wolfSSL with Arduino -##### Reformatting wolfSSL as a compatible Arduino Library -This is a shell script that will re-organize the wolfSSL library to be -compatible with Arduino projects that use Arduino IDE 1.5.0 or newer. -The Arduino IDE requires a library's source files to be in the library's root -directory with a header file in the name of the library. This script moves all -src/ files to the `IDE/ARDUINO/wolfSSL/src` directory and creates a stub header -file called `wolfssl.h` inside that directory. +See the [example sketches](./sketches/README.md): -Step 1: To configure wolfSSL with Arduino, enter the following from within the -wolfssl/IDE/ARDUINO directory: +- [sketches/wolfssl_server](./sketches/wolfssl_server/README.md) +- [sketches/wolfssl_client](./sketches/wolfssl_client/README.md) - `./wolfssl-arduino.sh` +When publishing a new version to the Arduino Registry, be sure to edit `WOLFSSL_VERSION_ARUINO_SUFFIX` in the `wolfssl-arduino.sh` script. -Step 2: Copy the directory wolfSSL that was just created to: -`~/Documents/Arduino/libraries/` directory so the Arduino IDE can find it. +## Boards -Step 3: Edit `/wolfSSL/src/user_settings.h` +Many of the supported boards are natively built-in to the [Arduino IDE Board Manager](https://docs.arduino.cc/software/ide-v2/tutorials/ide-v2-board-manager/) +and by adding [additional cores](https://docs.arduino.cc/learn/starting-guide/cores/) as needed. + +STM32 Support can be added by including this link in the "Additional Boards Managers URLs" field +from [stm32duino/Arduino_Core_STM32](https://github.com/stm32duino/Arduino_Core_STM32?tab=readme-ov-file#getting-started) . + +``` +https://github.com/stm32duino/BoardManagerFiles/raw/main/package_stmicroelectronics_index.json +``` + +## Using wolfSSL from the Arduino IDE + +The Official wolfSSL: https://github.com/wolfSSL/arduino-wolfSSL See [PR #1](https://github.com/wolfSSL/Arduino-wolfSSL/pull/1). + +This option will allow wolfSSL to be installed directly using the native Arduino tools. + +## Manually Reformatting wolfSSL as a Compatible Arduino Library + +Use [this](./wolfssl-arduino.sh) shell script that will re-organize the wolfSSL library to be +compatible with [Arduino Library Specification](https://arduino.github.io/arduino-cli/0.35/library-specification/) +for projects that use Arduino IDE 1.5.0 or newer. + +The Arduino IDE requires a library's source files to be in the library's root directory with a +header file in the name of the library. This script moves all `src/` files to the `IDE/ARDUINO/wolfSSL/src` +directory and creates a stub header file called `wolfssl.h` inside that directory. + +### Step 1: + +To configure wolfSSL with Arduino, enter ONE of the following 4 commands +from within the `wolfssl/IDE/ARDUINO` directory: + +1. `./wolfssl-arduino.sh` + - Creates an Arduino Library directory structure in the local `wolfSSL` directory of `IDE/ARDUINO`. + - You can add your own `user_settings.h`, or copy/rename the [default](../../examples/configs/user_settings_arduino.h). + +2. `./wolfssl-arduino.sh INSTALL` (The most common option) + - Creates an Arduino Library in the local `wolfSSL` directory + - Moves that directory to the Arduino library directory: + - `$HOME/Arduino/libraries` for most bash environments + - `/mnt/c/Users/$USER/Documents/Arduino/libraries` (for WSL) + - Adds the [default](../../examples/configs/user_settings_arduino.h) as `user_settings.h`. + - The wolfSSL library is now available from the Arduino IDE. + +3. `./wolfssl-arduino.sh INSTALL /path/to/repository` (Used to update [arduino-wolfSSL](https://github.com/wolfSSL/arduino-wolfSSL)) + - Creates an Arduino Library in `wolfSSL` directory + - Copies that directory contents to the specified `/path/to/repository` + - Adds the [default](../../examples/configs/user_settings_arduino.h) as `user_settings.h`. + +4. `./wolfssl-arduino.sh INSTALL /path/to/any/other/directory` + - Creates an Arduino Library in `wolfSSL` directory + - Copies that directory contents to the specified `/path/to/any/other/directory` + +### Step 2: + +Edit `/wolfSSL/src/user_settings.h` If building for Intel Galileo platform add: `#define INTEL_GALILEO`. -Add any other custom settings, for a good start see the examples in wolfssl root -"/examples/configs/user_settings_*.h" +Add any other custom settings. For a good start see the examples in wolfssl root +"[/examples/configs/user_settings_*.h](https://github.com/wolfssl/wolfssl/tree/master/examples/configs)" -Step 4: If you experience any issues with custom user_settings.h see the wolfssl +### Step 3: + +If you experience any issues with custom `user_settings.h` see the wolfssl porting guide here for more assistance: https://www.wolfssl.com/docs/porting-guide/ -Step 5: If you still have any issues contact support@wolfssl.com for more help. +If you have any issues contact support@wolfssl.com for help. + +# Including wolfSSL in Arduino Libraries (for Arduino version 2.0 or greater) + +1. In the Arduino IDE: + +The wolfSSL library should automatically be detected when found in the `libraries` +directory. + + - In `Sketch -> Include Library` choose wolfSSL for new sketches. + ##### Including wolfSSL in Arduino Libraries (for Arduino version 1.6.6) @@ -33,6 +92,90 @@ Step 5: If you still have any issues contact support@wolfssl.com for more help. `IDE/ARDUNIO/wolfSSL` folder. - In `Sketch -> Include Library` choose wolfSSL. -2. Open an example Arduino sketch for wolfSSL: - - wolfSSL Client INO sketch: `sketches/wolfssl_client/wolfssl_client.ino` - - wolfSSL Server INO sketch: `sketches/wolfssl_server/wolfssl_server.ino` +##### wolfSSL Examples + +Open an example Arduino sketch for wolfSSL: + + - wolfSSL [Client INO sketch](./sketches/wolfssl_client/README.md): `sketches/wolfssl_client/wolfssl_client.ino` + + - wolfSSL [Server INO sketch](./sketches/wolfssl_server/README.md): `sketches/wolfssl_server/wolfssl_server.ino` + +#### Script Examples + +Refresh the local Windows Arduino wolfSSL library from GitHub repository directory using WSL: + +Don't forget to edit `WOLFSSL_VERSION_ARUINO_SUFFIX`! + +```bash +# Change to the wolfSSL Arduino IDE directory +cd /mnt/c/workspace/wolfssl-$USER/IDE/ARDUINO + +# remove current Arduino wolfSSL library +rm -rf /mnt/c/Users/$USER/Documents/Arduino/libraries/wolfssl + +# Install wolfSSL as an Arduino library +./wolfssl-arduino.sh INSTALL +``` + +Publish wolfSSL from WSL to a `Arduino-wolfSSL-$USER` repository. + +```bash +cd /mnt/c/workspace/wolfssl-$USER/IDE/ARDUINO +rm -rf /mnt/c/Users/$USER/Documents/Arduino/libraries/wolfSSL +rm -rf /mnt/c/workspace/wolfssl-$USER/IDE/ARDUINO/wolfSSL +./wolfssl-arduino.sh INSTALL /mnt/c/workspace/Arduino-wolfSSL-$USER/ +``` + +Publish wolfSSL from WSL to default Windows local library. + +```bash +cd /mnt/c/workspace/wolfssl-$USER/IDE/ARDUINO +rm -rf /mnt/c/Users/$USER/Documents/Arduino/libraries/wolfSSL +rm -rf /mnt/c/workspace/wolfssl-arduino/IDE/ARDUINO/wolfSSL +./wolfssl-arduino.sh INSTALL +``` + +Test the TLS server by running a local command-line client. + +```bash +cd /mnt/c/workspace/wolfssl-$USER +./examples/client/client -h 192.168.1.43 -p 11111 -v 3 +``` + +Build wolfSSL to include wolfSSH support to an alternate development directory. + +```bash +cd /mnt/c/workspace/wolfssl-$USER +./configure --prefix=/mnt/c/workspace/wolfssh-$USER/wolfssl_install --enable-ssh +make +make install + +``` + +Build wolfSSH with wolfSSL not installed to default directory. + +```bash +cd /mnt/c/workspace/wolfssh-$USER +./configure --with-wolfssl=/mnt/c/workspace/wolfssh-$USER/wolfssl_install +make +./examples/client/client -u jill -h 192.168.1.34 -p 22222 -P upthehill +``` + +Test the current wolfSSL. + +```bash +cd /mnt/c/workspace/wolfssl-arduino +git status +./autogen.sh +./configure --enable-all +make clean +make && make test +``` + +Build and run `testwolfcrypt`. + +```bash +./autogen.sh +./configure --enable-all +make clean && make && ./wolfcrypt/test/testwolfcrypt +``` diff --git a/extra/wolfssl/wolfssl/IDE/ARDUINO/include.am b/extra/wolfssl/wolfssl/IDE/ARDUINO/include.am index 19189e82..52491a8b 100644 --- a/extra/wolfssl/wolfssl/IDE/ARDUINO/include.am +++ b/extra/wolfssl/wolfssl/IDE/ARDUINO/include.am @@ -3,6 +3,15 @@ # All paths should be given relative to the root EXTRA_DIST+= IDE/ARDUINO/README.md +EXTRA_DIST+= IDE/ARDUINO/Arduino_README_prepend.md +EXTRA_DIST+= IDE/ARDUINO/keywords.txt +EXTRA_DIST+= IDE/ARDUINO/library.properties.template +EXTRA_DIST+= IDE/ARDUINO/sketches/README.md +EXTRA_DIST+= IDE/ARDUINO/sketches/wolfssl_client/README.md EXTRA_DIST+= IDE/ARDUINO/sketches/wolfssl_client/wolfssl_client.ino +EXTRA_DIST+= IDE/ARDUINO/sketches/wolfssl_server/README.md EXTRA_DIST+= IDE/ARDUINO/sketches/wolfssl_server/wolfssl_server.ino +EXTRA_DIST+= IDE/ARDUINO/sketches/wolfssl_version/README.md +EXTRA_DIST+= IDE/ARDUINO/sketches/wolfssl_version/wolfssl_version.ino +EXTRA_DIST+= IDE/ARDUINO/wolfssl.h EXTRA_DIST+= IDE/ARDUINO/wolfssl-arduino.sh diff --git a/extra/wolfssl/wolfssl/IDE/ARDUINO/keywords.txt b/extra/wolfssl/wolfssl/IDE/ARDUINO/keywords.txt new file mode 100644 index 00000000..27d5dc3d --- /dev/null +++ b/extra/wolfssl/wolfssl/IDE/ARDUINO/keywords.txt @@ -0,0 +1,21 @@ +# Syntax Coloring Map For wolfSSL +# See https://arduino.github.io/arduino-cli/0.35/library-specification/#keywords +# +# Be sure to use tabs, not spaces. This might help: +# tr ' ' '\t' < keywords1.txt > keywords.txt + +#============================================= +# Datatypes (KEYWORD1) +#============================================= + + +#============================================= +# Methods and Functions (KEYWORD2) +#============================================= +wolfSSL_SetIORecv KEYWORD1 + +#============================================= +# Instances (KEYWORD2) +#============================================= +ctx KEYWORD2 + diff --git a/extra/wolfssl/wolfssl/IDE/ARDUINO/library.properties.template b/extra/wolfssl/wolfssl/IDE/ARDUINO/library.properties.template new file mode 100644 index 00000000..9b18e8c2 --- /dev/null +++ b/extra/wolfssl/wolfssl/IDE/ARDUINO/library.properties.template @@ -0,0 +1,9 @@ +name=wolfssl +version=${WOLFSSL_VERSION}${WOLFSSL_VERSION_ARUINO_SUFFIX} +author=wolfSSL Inc. +maintainer=wolfSSL inc +sentence=A lightweight SSL/TLS library written in ANSI C and targeted for embedded, RTOS, and resource-constrained environments. +paragraph=Manual: https://www.wolfssl.com/documentation/manuals/wolfssl/index.html. +category=Communication +url=https://www.wolfssl.com/ +architectures=* diff --git a/extra/wolfssl/wolfssl/IDE/ARDUINO/sketches/README.md b/extra/wolfssl/wolfssl/IDE/ARDUINO/sketches/README.md new file mode 100644 index 00000000..26a1d1e1 --- /dev/null +++ b/extra/wolfssl/wolfssl/IDE/ARDUINO/sketches/README.md @@ -0,0 +1,12 @@ +# wolfSSL Arduino Examples + +There are currently two example Arduino sketches: + +* [wolfssl_client](./wolfssl_client/README.md): Basic TLS listening client. +* [wolfssl_server](./wolfssl_server/README.md): Basic TLS server. + +Examples have been most recently confirmed operational on the +[Arduino IDE](https://www.arduino.cc/en/software) 2.2.1. + +For examples on other platforms, see the [IDE directory](https://github.com/wolfssl/wolfssl/tree/master/IDE). +Additional examples can be found on [wolfSSL/wolfssl-examples](https://github.com/wolfSSL/wolfssl-examples/). diff --git a/extra/wolfssl/wolfssl/IDE/ARDUINO/sketches/wolfssl_client/README.md b/extra/wolfssl/wolfssl/IDE/ARDUINO/sketches/wolfssl_client/README.md new file mode 100644 index 00000000..caf83c58 --- /dev/null +++ b/extra/wolfssl/wolfssl/IDE/ARDUINO/sketches/wolfssl_client/README.md @@ -0,0 +1,22 @@ +# Arduino Basic TLS Listening Client + +Open the [wolfssl_client.ino](./wolfssl_client.ino) file in the Arduino IDE. + +Other IDE products are also supported, such as: + +- [PlatformIO in VS Code](https://docs.platformio.org/en/latest/frameworks/arduino.html) +- [VisualGDB](https://visualgdb.com/tutorials/arduino/) +- [VisualMicro](https://www.visualmicro.com/) + +For examples on other platforms, see the [IDE directory](https://github.com/wolfssl/wolfssl/tree/master/IDE). +Additional examples can be found on [wolfSSL/wolfssl-examples](https://github.com/wolfSSL/wolfssl-examples/). + + +### Troubleshooting + +When encountering odd errors such as `undefined reference to ``_impure_ptr'`, try cleaning the Arduino +cache directories. For Windows, that's typically in: + +```text +C:\Users\%USERNAME%\AppData\Local\Temp\arduino\sketches +``` diff --git a/extra/wolfssl/wolfssl/IDE/ARDUINO/sketches/wolfssl_client/wolfssl_client.ino b/extra/wolfssl/wolfssl/IDE/ARDUINO/sketches/wolfssl_client/wolfssl_client.ino index 61362ae9..21a84deb 100644 --- a/extra/wolfssl/wolfssl/IDE/ARDUINO/sketches/wolfssl_client/wolfssl_client.ino +++ b/extra/wolfssl/wolfssl/IDE/ARDUINO/sketches/wolfssl_client/wolfssl_client.ino @@ -20,157 +20,875 @@ */ /* - This was original tested with Intel Galileo acting as the Client, with a -laptop acting as a server using the server example provided in examples/server. -Legacy Ardunio v1.86 was used to compile and program the Galileo +Tested with: + +1) Intel Galileo acting as the Client, with a laptop acting as a server using + the server example provided in examples/server. + Legacy Arduino v1.86 was used to compile and program the Galileo + +2) Espressif ESP32 WiFi + +3) Arduino Due, Nano33 IoT, Nano RP-2040 */ -#define USE_CERT_BUFFERS_2048 +/* + * Note to code editors: the Arduino client and server examples are edited in + * parallel for side-by-side comparison between examples. + */ + +/* If you have a private include, define it here, otherwise edit WiFi params */ +#define MY_PRIVATE_CONFIG "/workspace/my_private_config.h" + +/* set REPEAT_CONNECTION to a non-zero value to continually run the example. */ +#define REPEAT_CONNECTION 0 + +/* Edit this with your other TLS host server address to connect to: */ +#define WOLFSSL_TLS_SERVER_HOST "192.168.1.39" + +/* wolfssl TLS examples communicate on port 11111 */ +#define WOLFSSL_PORT 11111 + +/* Choose a monitor serial baud rate: 9600, 14400, 19200, 57600, 74880, etc. */ +#define SERIAL_BAUD 115200 + +/* We'll wait up to 2000 milliseconds to properly shut down connection */ +#define SHUTDOWN_DELAY_MS 2000 + +/* Number of times to retry connection. */ +#define RECONNECT_ATTEMPTS 20 + +/* Optional stress test. Define to consume memory until exhausted: */ +/* #define MEMORY_STRESS_TEST */ + +/* Choose client or server example, not both. */ +#define WOLFSSL_CLIENT_EXAMPLE +/* #define WOLFSSL_SERVER_EXAMPLE */ + +#if defined(MY_PRIVATE_CONFIG) + /* the /workspace directory may contain a private config + * excluded from GitHub with items such as WiFi passwords */ + #include MY_PRIVATE_CONFIG + static const char* ssid PROGMEM = MY_ARDUINO_WIFI_SSID; + static const char* password PROGMEM = MY_ARDUINO_WIFI_PASSWORD; +#else + /* when using WiFi capable boards: */ + static const char* ssid PROGMEM = "your_SSID"; + static const char* password PROGMEM = "your_PASSWORD"; +#endif + +#define BROADCAST_ADDRESS "255.255.255.255" + +/* There's an optional 3rd party NTPClient library by Fabrice Weinberg. + * If it is installed, uncomment define USE_NTP_LIB here: */ +/* #define USE_NTP_LIB */ +#ifdef USE_NTP_LIB + #include +#endif + #include +/* Important: make sure settings.h appears before any other wolfSSL headers */ +#include +/* Reminder: settings.h includes user_settings.h + * For ALL project wolfSSL settings, see: + * [your path]/Arduino\libraries\wolfSSL\src\user_settings.h */ #include -#include #include +#include + +/* Define DEBUG_WOLFSSL in user_settings.h for more verbose logging. */ +#if defined(DEBUG_WOLFSSL) + #define PROGRESS_DOT F("") +#else + #define PROGRESS_DOT F(".") +#endif + +/* Convert a macro to a string */ +#define xstr(x) str(x) +#define str(x) #x + +/* optional board-specific networking includes */ +#if defined(ESP32) + #define USING_WIFI + #include + #include + #ifdef USE_NTP_LIB + WiFiUDP ntpUDP; + #endif + /* Ensure the F() flash macro is defined */ + #ifndef F + #define F + #endif + WiFiClient client; + +#elif defined(ESP8266) + #define USING_WIFI + #include + WiFiClient client; + +#elif defined(ARDUINO_SAM_DUE) + #include + /* There's no WiFi/Ethernet on the Due. Requires Ethernet Shield. + /* Needs "Ethernet by Various" library to be installed. Tested with V2.0.2 */ + #include + EthernetClient client; + +#elif defined(ARDUINO_SAMD_NANO_33_IOT) + #define USING_WIFI + #include + #include /* Needs Arduino WiFiNINA library installed manually */ + WiFiClient client; + +#elif defined(ARDUINO_ARCH_RP2040) + #define USING_WIFI + #include + #include + WiFiClient client; + +#elif defined(USING_WIFI) + #define USING_WIFI + #include + #include + #ifdef USE_NTP_LIB + WiFiUDP ntpUDP; + #endif + WiFiClient client; + +/* TODO +#elif defined(OTHER_BOARD) +*/ +#else + #define USING_WIFI + WiFiClient client; + +#endif + +/* Only for syntax highlighters to show interesting options enabled: */ +#if defined(HAVE_SNI) \ + || defined(HAVE_MAX_FRAGMENT) \ + || defined(HAVE_TRUSTED_CA) \ + || defined(HAVE_TRUNCATED_HMAC) \ + || defined(HAVE_CERTIFICATE_STATUS_REQUEST) \ + || defined(HAVE_CERTIFICATE_STATUS_REQUEST_V2) \ + || defined(HAVE_SUPPORTED_CURVES) \ + || defined(HAVE_ALPN) \ + || defined(HAVE_SESSION_TICKET) \ + || defined(HAVE_SECURE_RENEGOTIATION) \ + || defined(HAVE_SERVER_RENEGOTIATION_INFO) +#endif + +static const char host[] PROGMEM = WOLFSSL_TLS_SERVER_HOST; /* server to connect to */ +static const int port PROGMEM = WOLFSSL_PORT; /* port on server to connect to */ + +static WOLFSSL_CTX* ctx = NULL; +static WOLFSSL* ssl = NULL; +static char* wc_error_message = (char*)malloc(80 + 1); +static char errBuf[80]; + +#if defined(MEMORY_STRESS_TEST) + #define MEMORY_STRESS_ITERATIONS 100 + #define MEMORY_STRESS_BLOCK_SIZE 1024 + #define MEMORY_STRESS_INITIAL (4*1024) + static char* memory_stress[MEMORY_STRESS_ITERATIONS]; /* typically 1K per item */ + static int mem_ctr = 0; +#endif + +static int EthernetSend(WOLFSSL* ssl, char* msg, int sz, void* ctx); +static int EthernetReceive(WOLFSSL* ssl, char* reply, int sz, void* ctx); +static int reconnect = RECONNECT_ATTEMPTS; +static int lng_index PROGMEM = 0; /* 0 = English */ + +#if defined(__arm__) + #include + extern char _end; + extern "C" char *sbrk(int i); + static char *ramstart=(char *)0x20070000; + static char *ramend=(char *)0x20088000; +#endif + +/*****************************************************************************/ +/* fail_wait - in case of unrecoverable error */ +/*****************************************************************************/ +int fail_wait(void) { + show_memory(); + + Serial.println(F("Failed. Halt.")); + while (1) { + delay(1000); + } + return 0; +} + +/*****************************************************************************/ +/* show_memory() to optionally view during debugging. */ +/*****************************************************************************/ +int show_memory(void) +{ +#if defined(__arm__) + struct mallinfo mi = mallinfo(); + + char *heapend=sbrk(0); + register char * stack_ptr asm("sp"); + #if defined(DEBUG_WOLFSSL_VERBOSE) + Serial.print(" arena="); + Serial.println(mi.arena); + Serial.print(" ordblks="); + Serial.println(mi.ordblks); + Serial.print(" uordblks="); + Serial.println(mi.uordblks); + Serial.print(" fordblks="); + Serial.println(mi.fordblks); + Serial.print(" keepcost="); + Serial.println(mi.keepcost); + #endif + + #if defined(DEBUG_WOLFSSL) || defined(MEMORY_STRESS_TEST) + Serial.print("Estimated free memory: "); + Serial.print(stack_ptr - heapend + mi.fordblks); + Serial.println(F(" bytes")); + #endif + + #if (0) + /* Experimental: not supported on all devices: */ + Serial.print("RAM Start %lx\n", (unsigned long)ramstart); + Serial.print("Data/Bss end %lx\n", (unsigned long)&_end); + Serial.print("Heap End %lx\n", (unsigned long)heapend); + Serial.print("Stack Ptr %lx\n",(unsigned long)stack_ptr); + Serial.print("RAM End %lx\n", (unsigned long)ramend); + + Serial.print("Heap RAM Used: ",mi.uordblks); + Serial.print("Program RAM Used ",&_end - ramstart); + Serial.print("Stack RAM Used ",ramend - stack_ptr); + + Serial.print("Estimated Free RAM: %d\n\n",stack_ptr - heapend + mi.fordblks); + #endif +#else + Serial.println(F("show_memory() not implemented for this platform")); +#endif + return 0; +} + +/*****************************************************************************/ +/* EthernetSend() to send a message string. */ +/*****************************************************************************/ +int EthernetSend(WOLFSSL* ssl, char* message, int sz, void* ctx) { + int sent = 0; + (void)ssl; + (void)ctx; + + sent = client.write((byte*)message, sz); + return sent; +} + +/*****************************************************************************/ +/* EthernetReceive() to receive a reply string. */ +/*****************************************************************************/ +int EthernetReceive(WOLFSSL* ssl, char* reply, int sz, void* ctx) { + int ret = 0; + (void)ssl; + (void)ctx; + + while (client.available() > 0 && ret < sz) { + reply[ret++] = client.read(); + } + return ret; +} + +/*****************************************************************************/ +/* Arduino setup_hardware() */ +/*****************************************************************************/ +int setup_hardware(void) { + int ret = 0; + +#if defined(ARDUINO_SAMD_NANO_33_IOT) + Serial.println(F("Detected known tested and working Arduino Nano 33 IoT")); +#elif defined(ARDUINO_ARCH_RP2040) + Serial.println(F("Detected known tested and working Arduino RP-2040")); +#elif defined(__arm__) && defined(ID_TRNG) && defined(TRNG) + /* need to manually turn on random number generator on Arduino Due, etc. */ + pmc_enable_periph_clk(ID_TRNG); + trng_enable(TRNG); + Serial.println(F("Enabled ARM TRNG")); +#endif + + show_memory(); + randomSeed(analogRead(0)); + return ret; +} + +/*****************************************************************************/ +/* Arduino setup_datetime() */ +/* The device needs to have a valid date within the valid range of certs. */ +/*****************************************************************************/ +int setup_datetime(void) { + int ret = 0; + int ntp_tries = 20; + + /* we need a date in the range of cert expiration */ +#ifdef USE_NTP_LIB + #if defined(ESP32) + NTPClient timeClient(ntpUDP, "pool.ntp.org"); + + timeClient.begin(); + timeClient.update(); + delay(1000); + while (!timeClient.isTimeSet() && (ntp_tries > 0)) { + timeClient.forceUpdate(); + Serial.println(F("Waiting for NTP update")); + delay(2000); + ntp_tries--; + } + if (ntp_tries <= 0) { + Serial.println(F("Warning: gave up waiting on NTP")); + } + Serial.println(timeClient.getFormattedTime()); + Serial.println(timeClient.getEpochTime()); + #endif +#endif + +#if defined(ESP32) + /* see esp32-hal-time.c */ + ntp_tries = 5; + /* Replace "pool.ntp.org" with your preferred NTP server */ + configTime(0, 0, "pool.ntp.org"); + + /* Wait for time to be set */ + while ((time(nullptr) <= 100000) && ntp_tries > 0) { + Serial.println(F("Waiting for time to be set...")); + delay(2000); + ntp_tries--; + } +#endif + return ret; +} /* setup_datetime */ + +/*****************************************************************************/ +/* Arduino setup_network() */ +/*****************************************************************************/ +int setup_network(void) { + int ret = 0; -const char host[] = "192.168.1.148"; /* server to connect to */ -const int port = 11111; /* port on server to connect to */ +#if defined(USING_WIFI) + int status = WL_IDLE_STATUS; -int EthernetSend(WOLFSSL* ssl, char* msg, int sz, void* ctx); -int EthernetReceive(WOLFSSL* ssl, char* reply, int sz, void* ctx); -int reconnect = 10; + /* The ESP8266 & ESP32 support both AP and STA. We'll use STA: */ + #if defined(ESP8266) || defined(ESP32) + WiFi.mode(WIFI_STA); + #else + String fv; + if (WiFi.status() == WL_NO_MODULE) { + Serial.println("Communication with WiFi module failed!"); + /* don't continue if no network */ + while (true) ; + } -EthernetClient client; + fv = WiFi.firmwareVersion(); + if (fv < WIFI_FIRMWARE_LATEST_VERSION) { + Serial.println("Please upgrade the firmware"); + } + #endif + + Serial.print(F("Connecting to WiFi ")); + Serial.print(ssid); + status = WiFi.begin(ssid, password); + while (status != WL_CONNECTED) { + delay(1000); + Serial.print(F(".")); + Serial.print(status); + status = WiFi.status(); + } + + Serial.println(F(" Connected!")); +#else + /* Newer Ethernet shields have a + * MAC address printed on a sticker on the shield */ + byte mac[] = { 0xDE, 0xAD, 0xBE, 0xEF, 0xFE, 0xED }; + IPAddress ip(192, 168, 1, 42); + IPAddress myDns(192, 168, 1, 1); + Ethernet.init(10); /* Most Arduino shields */ + /* Ethernet.init(5); * MKR ETH Shield */ + /* Ethernet.init(0); * Teensy 2.0 */ + /* Ethernet.init(20); * Teensy++ 2.0 */ + /* Ethernet.init(15); * ESP8266 with Adafruit FeatherWing Ethernet */ + /* Ethernet.init(33); * ESP32 with Adafruit FeatherWing Ethernet */ + Serial.println(F("Initialize Ethernet with DHCP:")); + if (Ethernet.begin(mac) == 0) { + Serial.println(F("Failed to configure Ethernet using DHCP")); + /* Check for Ethernet hardware present */ + if (Ethernet.hardwareStatus() == EthernetNoHardware) { + Serial.println(F("Ethernet shield was not found.")); + while (true) { + delay(1); /* do nothing */ + } + } + if (Ethernet.linkStatus() == LinkOFF) { + Serial.println(F("Ethernet cable is not connected.")); + } + /* try to configure using IP address instead of DHCP : */ + Ethernet.begin(mac, ip, myDns); + } + else { + Serial.print(F(" DHCP assigned IP ")); + Serial.println(Ethernet.localIP()); + } + /* We'll assume the Ethernet connection is ready to go. */ +#endif -WOLFSSL_CTX* ctx = NULL; -WOLFSSL* ssl = NULL; + Serial.println(F("********************************************************")); + Serial.print(F(" wolfSSL Example Client IP = ")); +#if defined(USING_WIFI) + Serial.println(WiFi.localIP()); +#else + Serial.println(Ethernet.localIP()); +#endif + Serial.print(F(" Configured Server Host to connect to: ")); + Serial.println(host); + Serial.println(F("********************************************************")); + Serial.println(F("Setup network complete.")); -void setup() { + return ret; +} + +/*****************************************************************************/ +/* Arduino setup_wolfssl() */ +/*****************************************************************************/ +int setup_wolfssl(void) { + int ret = 0; WOLFSSL_METHOD* method; - /* Initialize Return Code */ - int rc; - Serial.begin(9600); - /* Delay need to ensure connection to server */ - delay(4000); - method = wolfTLSv1_2_client_method(); + /* Show a revision of wolfssl user_settings.h file in use when available: */ +#if defined(WOLFSSL_USER_SETTINGS_ID) + Serial.print(F("WOLFSSL_USER_SETTINGS_ID: ")); + Serial.println(F(WOLFSSL_USER_SETTINGS_ID)); +#else + Serial.println(F("No WOLFSSL_USER_SETTINGS_ID found.")); +#endif + +#if defined(NO_WOLFSSL_SERVER) + Serial.println(F("wolfSSL server code disabled to save space.")); +#endif +#if defined(NO_WOLFSSL_CLIENT) + Serial.println(F("wolfSSL client code disabled to save space.")); +#endif + +#if defined(DEBUG_WOLFSSL) + wolfSSL_Debugging_ON(); + Serial.println(F("wolfSSL Debugging is On!")); +#else + Serial.println(F("wolfSSL Debugging is Off! (enable with DEBUG_WOLFSSL)")); +#endif + + /* See ssl.c for TLS cache settings. Larger cache = use more RAM. */ +#if defined(NO_SESSION_CACHE) + Serial.println(F("wolfSSL TLS NO_SESSION_CACHE")); +#elif defined(MICRO_SESSION_CACHEx) + Serial.println(F("wolfSSL TLS MICRO_SESSION_CACHE")); +#elif defined(SMALL_SESSION_CACHE) + Serial.println(F("wolfSSL TLS SMALL_SESSION_CACHE")); +#elif defined(MEDIUM_SESSION_CACHE) + Serial.println(F("wolfSSL TLS MEDIUM_SESSION_CACHE")); +#elif defined(BIG_SESSION_CACHE) + Serial.println(F("wolfSSL TLS BIG_SESSION_CACHE")); +#elif defined(HUGE_SESSION_CACHE) + Serial.println(F("wolfSSL TLS HUGE_SESSION_CACHE")); +#elif defined(HUGE_SESSION_CACHE) + Serial.println(F("wolfSSL TLS HUGE_SESSION_CACHE")); +#else + Serial.println(F("WARNING: Unknown or no TLS session cache setting.")); + /* See wolfssl/src/ssl.c for amount of memory used. + * It is best on embedded devices to choose a TLS session cache size. */ +#endif + + ret = wolfSSL_Init(); + if (ret == WOLFSSL_SUCCESS) { + Serial.println("Successfully called wolfSSL_Init"); + } + else { + Serial.println("ERROR: wolfSSL_Init failed"); + } + + /* See companion server example with wolfSSLv23_server_method here. + * method = wolfSSLv23_client_method()); SSL 3.0 - TLS 1.3. + * method = wolfTLSv1_2_client_method(); only TLS 1.2 + * method = wolfTLSv1_3_client_method(); only TLS 1.3 + * + * see Arduino\libraries\wolfssl\src\user_settings.h */ + + Serial.println("Here we go!"); + + method = wolfSSLv23_client_method(); if (method == NULL) { - Serial.println("unable to get method"); - return; + Serial.println(F("unable to get wolfssl client method")); + fail_wait(); } ctx = wolfSSL_CTX_new(method); if (ctx == NULL) { - Serial.println("unable to get ctx"); - return; + Serial.println(F("unable to get ctx")); + fail_wait(); } - /* initialize wolfSSL using callback functions */ + + return ret; +} + +/*****************************************************************************/ +/* Arduino setup_certificates() */ +/*****************************************************************************/ +int setup_certificates(void) { + int ret = 0; + + Serial.println(F("Initializing certificates...")); + show_memory(); + + /* Use built-in validation, No verification callback function: */ wolfSSL_CTX_set_verify(ctx, SSL_VERIFY_PEER, 0); - rc = wolfSSL_CTX_load_verify_buffer(ctx, ca_cert_der_2048,\ - sizeof_ca_cert_der_2048,\ - WOLFSSL_FILETYPE_ASN1); - Serial.print("\n\n Return code of load_verify is:"); - Serial.println(rc); - Serial.println(""); - rc = wolfSSL_CTX_use_certificate_buffer(ctx, client_cert_der_2048,\ - sizeof_client_cert_der_2048,\ - WOLFSSL_FILETYPE_ASN1); - Serial.print("\n\n Return code of use_certificate_buffer is:"); - Serial.println(rc); - Serial.println(""); - rc = wolfSSL_CTX_use_PrivateKey_buffer(ctx, client_key_der_2048,\ - sizeof_client_key_der_2048,\ - WOLFSSL_FILETYPE_ASN1); - Serial.print("\n\n Return code of use_PrivateKey_buffer is:"); - Serial.println(rc); - Serial.println(""); + + /* Certificate */ + Serial.println("Initializing certificates..."); + ret = wolfSSL_CTX_use_certificate_buffer(ctx, + CTX_CLIENT_CERT, + CTX_CLIENT_CERT_SIZE, + CTX_CLIENT_CERT_TYPE); + if (ret == WOLFSSL_SUCCESS) { + Serial.print("Success: use certificate: "); + Serial.println(xstr(CTX_SERVER_CERT)); + } + else { + Serial.println(F("Error: wolfSSL_CTX_use_certificate_buffer failed: ")); + wc_ErrorString(ret, wc_error_message); + Serial.println(wc_error_message); + fail_wait(); + } + + /* Setup private client key */ + ret = wolfSSL_CTX_use_PrivateKey_buffer(ctx, + CTX_CLIENT_KEY, + CTX_CLIENT_KEY_SIZE, + CTX_CLIENT_KEY_TYPE); + if (ret == WOLFSSL_SUCCESS) { + Serial.print("Success: use private key buffer: "); + Serial.println(xstr(CTX_SERVER_KEY)); + } + else { + Serial.println(F("Error: wolfSSL_CTX_use_PrivateKey_buffer failed: ")); + wc_ErrorString(ret, wc_error_message); + Serial.println(wc_error_message); + fail_wait(); + } + + ret = wolfSSL_CTX_load_verify_buffer(ctx, + CTX_CA_CERT, + CTX_CA_CERT_SIZE, + CTX_CA_CERT_TYPE); + if (ret == WOLFSSL_SUCCESS) { + Serial.println(F("Success: load_verify CTX_CA_CERT")); + } + else { + Serial.println(F("Error: wolfSSL_CTX_load_verify_buffer failed: ")); + wc_ErrorString(ret, wc_error_message); + Serial.println(wc_error_message); + fail_wait(); + } + + + + return ret; +} /* Arduino setup */ + +/*****************************************************************************/ +/*****************************************************************************/ +/* Arduino setup() */ +/*****************************************************************************/ +/*****************************************************************************/ +void setup(void) { + int i = 0; + Serial.begin(SERIAL_BAUD); + while (!Serial && (i < 10)) { + /* wait for serial port to connect. Needed for native USB port only */ + delay(1000); + i++; + } + Serial.println(F("")); + Serial.println(F("")); + Serial.println(F("wolfSSL TLS Client Example Startup.")); + + /* define DEBUG_WOLFSSL in wolfSSL user_settings.h for diagnostics */ +#if defined(DEBUG_WOLFSSL) + wolfSSL_Debugging_ON(); +#endif + + /* Optionally pre-allocate a large block of memory for testing */ +#if defined(MEMORY_STRESS_TEST) + Serial.println(F("WARNING: Memory Stress Test Active!")); + Serial.print(F("Allocating extra memory: ")); + Serial.print(MEMORY_STRESS_INITIAL); + Serial.println(F(" bytes...")); + memory_stress[mem_ctr] = (char*)malloc(MEMORY_STRESS_INITIAL); + show_memory(); +#endif + + setup_hardware(); + + setup_network(); + + setup_datetime(); + + setup_wolfssl(); + + setup_certificates(); + + /* Initialize wolfSSL using callback functions. */ wolfSSL_SetIOSend(ctx, EthernetSend); wolfSSL_SetIORecv(ctx, EthernetReceive); - return; -} -int EthernetSend(WOLFSSL* ssl, char* msg, int sz, void* ctx) { - int sent = 0; - sent = client.write((byte*)msg, sz); - return sent; -} + Serial.println(F("Completed Arduino setup!")); + /* See companion wolfssl_server.ino code; server begins listening here + * https://github.com/wolfSSL/wolfssl/tree/master/IDE/ARDUINO/sketches/wolfssl_server + * Any other server will work. See also: + * https://github.com/wolfSSL/wolfssl/tree/master/examples/client + */ + /* See companion wolfssl_server.ino code */ + return; +} /* Arduino setup */ -int EthernetReceive(WOLFSSL* ssl, char* reply, int sz, void* ctx) { +/*****************************************************************************/ +/* wolfSSL error_check() */ +/*****************************************************************************/ +int error_check(int this_ret, bool halt_on_error, + const __FlashStringHelper* message) { int ret = 0; - while (client.available() > 0 && ret < sz) { - reply[ret++] = client.read(); + if (this_ret == WOLFSSL_SUCCESS) { + Serial.print(F("Success: ")); + Serial.println(message); + } + else { + Serial.print(F("ERROR: return = ")); + Serial.print(this_ret); + Serial.print(F(": ")); + Serial.println(message); + Serial.println(wc_GetErrorString(this_ret)); + if (halt_on_error) { + fail_wait(); + } } + show_memory(); + return ret; +} /* error_check */ + +/*****************************************************************************/ +/* wolfSSL error_check_ssl */ +/* Parameters: */ +/* ssl is the current WOLFSSL object pointer */ +/* halt_on_error set to true to suspend operations for critical error */ +/* message is expected to be a memory-efficient F("") macro string */ +/*****************************************************************************/ +int error_check_ssl(WOLFSSL* ssl, int this_ret, bool halt_on_error, + const __FlashStringHelper* message) { + int err = 0; + + if (ssl == NULL) { + Serial.println(F("ssl is Null; Unable to allocate SSL object?")); +#ifndef DEBUG_WOLFSSL + Serial.println(F("Define DEBUG_WOLFSSL in user_settings.h for more.")); +#else + Serial.println(F("See wolfssl/wolfcrypt/error-crypt.h for codes.")); +#endif + Serial.print(F("ERROR: ")); + Serial.println(message); + show_memory(); + if (halt_on_error) { + fail_wait(); + } + } + else { + err = wolfSSL_get_error(ssl, this_ret); + if (err == WOLFSSL_SUCCESS) { + Serial.print(F("Success m: ")); + Serial.println(message); + } + else { + if (err < 0) { + wolfSSL_ERR_error_string(err, errBuf); + Serial.print(F("WOLFSSL Error: ")); + Serial.print(err); + Serial.print(F("; ")); + Serial.println(errBuf); + } + else { + Serial.println(F("Success: ssl object.")); + } + } + } + + return err; } +/*****************************************************************************/ +/*****************************************************************************/ +/* Arduino loop() */ +/*****************************************************************************/ +/*****************************************************************************/ void loop() { - int err = 0; - int input = 0; - int total_input = 0; - char msg[32] = "hello wolfssl!"; - int msgSz = (int)strlen(msg); - char errBuf[80]; char reply[80]; + char msg[32] = "hello wolfssl!"; const char* cipherName; + int retry_shutdown = SHUTDOWN_DELAY_MS; /* max try, once per millisecond */ + int total_input = 0; + int msgSz = 0; + int input = 0; + int ret = 0; + int err = 0; + msgSz = (int)strlen(msg); + Serial.println(F("")); + Serial.println(F("Starting Arduino loop() ...")); + if (reconnect) { reconnect--; - if (client.connect(host, port)) { - Serial.print("Connected to "); - Serial.println(host); + /* WiFi client returns true if connection succeeds, false if not. */ + /* Wired client returns int (1,-1,-2,-3,-4) for connection status. */ + Serial.print(F("Connecting to ")); + Serial.print(host); + Serial.print(F(":")); + Serial.println(port); + /* can also use: IPAddress server(192,168,1,37); */ + Serial.println(F("Here we go...")); + ret = client.connect(host, port); + Serial.println(F("Ok, checking...")); + if (ret > 0) { + Serial.println(F("Connected!")); + + /* initialize wolfSSL */ + ret = wolfSSL_Init(); + error_check(ret, false, F("calling wolfSSL_Init") ); + + /* create secure connection object. see setup for ctx certs. */ + Serial.println(F("Calling ssl = wolfSSL_new(ctx)")); ssl = wolfSSL_new(ctx); - if (ssl == NULL) { - Serial.println("Unable to allocate SSL object"); - return; - } - err = wolfSSL_connect(ssl); - if (err != WOLFSSL_SUCCESS) { - err = wolfSSL_get_error(ssl, 0); - wolfSSL_ERR_error_string(err, errBuf); - Serial.print("TLS Connect Error: "); - Serial.println(errBuf); - } - Serial.print("SSL version is "); + error_check_ssl(ssl, 0, true, F("Create WOLFSSL object from ctx")); + + Serial.print(F("Connecting to wolfSSL TLS Secure Server...")); + do { + err = 0; /* reset error */ + Serial.println(F("wolfSSL_connect ...")); + ret = wolfSSL_connect(ssl); + Serial.print("wolfSSL_connect return result ="); + Serial.println(ret); + if ((ret != WOLFSSL_SUCCESS) && (ret != WC_PENDING_E)) { + Serial.println(F("Failed connection, checking error.")); + err = error_check_ssl(ssl, ret, true, + F("Create WOLFSSL object from ctx")); + Serial.print("err ="); + Serial.println(err); + } + else { + Serial.print(PROGRESS_DOT); + } + } while (err == WC_PENDING_E); + + Serial.println(); + Serial.println(F("Connected!")); + Serial.print(F("SSL version is ")); Serial.println(wolfSSL_get_version(ssl)); + cipherName = wolfSSL_get_cipher(ssl); - Serial.print("SSL cipher suite is "); + Serial.print(F("SSL cipher suite is ")); Serial.println(cipherName); - if ((wolfSSL_write(ssl, msg, msgSz)) == msgSz) { - Serial.print("Server response: "); - /* wait for data */ - while (!client.available()) {} + + /* see test.h + * TODO: test.h needs a little bit of Arduino work for these: + showPeerEx(ssl, lng_index); + showPeerPEM(ssl); + */ + + Serial.print(F("Sending secure message to server: ")); + Serial.println(msg); + ret = wolfSSL_write(ssl, msg, msgSz); + if (ret == msgSz) { + Serial.print(F("Waiting for Server response...")); + + while (!client.available()) { + /* wait for data */ + delay(1); /* 1 ms delay */ + } + + Serial.print(F("Reading response..")); /* read data */ - while (wolfSSL_pending(ssl)) { - input = wolfSSL_read(ssl, reply, sizeof(reply) - 1); - total_input += input; - if (input < 0) { - err = wolfSSL_get_error(ssl, 0); - wolfSSL_ERR_error_string(err, errBuf); - Serial.print("TLS Read Error: "); - Serial.println(errBuf); - break; - } - else if (input > 0) { - reply[input] = '\0'; - Serial.print(reply); + do { + ret = wolfSSL_read(ssl, reply, sizeof(reply) - 1); + if (ret < 0) { + error_check_ssl(ssl, ret, false, + F("during TLS Read")); } else { - Serial.println(); + Serial.print(PROGRESS_DOT); } - } - } + } while (err == WC_PENDING_E); + Serial.println(); + + Serial.println(); + Serial.println(reply); /* typically: I hear you fa shizzle! */ + Serial.println(); + + } /* wolfSSL_write message size matched */ else { - err = wolfSSL_get_error(ssl, 0); - wolfSSL_ERR_error_string(err, errBuf); - Serial.print("TLS Write Error: "); - Serial.println(errBuf); + error_check_ssl(ssl, ret, false, + F("during TLS Write")); + } /* any wolfSSL_write message size mismatch is an error */ + + Serial.print(F("Shutting down..")); + do { + delay(1); + Serial.print(PROGRESS_DOT); + retry_shutdown--; + ret = wolfSSL_shutdown(ssl); + } while ( (ret == WOLFSSL_SHUTDOWN_NOT_DONE) + && (retry_shutdown > 0) + ); /* There may be pending data, so wait until done. */ + Serial.println(); + + if (retry_shutdown <= 0) { + /* if wolfSSL_free is called before properly shutting down the + * ssl object, undesired results may occur. */ + Serial.println(F("Warning! Shutdown did not properly complete.")); } - wolfSSL_shutdown(ssl); + wolfSSL_free(ssl); client.stop(); - Serial.println("Connection complete."); - reconnect = 0; - } + Serial.println(F("Connection complete.")); + if (REPEAT_CONNECTION) { + reconnect = RECONNECT_ATTEMPTS; + } + else { + reconnect = 0; + } + } /* client.connect(host, port) */ else { - Serial.println("Trying to reconnect..."); + Serial.println(F("Problem sending message. Trying to reconnect...")); } } delay(1000); -} + if ((reconnect > 0) && (REPEAT_CONNECTION)) { + Serial.println(F("Arduino loop repeating...")); + Serial.println(); + } + else { + printf("wow"); + Serial.println(F("Done!")); + while(1) { + /* wait forever */ + } + } + +#if defined(MEMORY_STRESS_TEST) + if (mem_ctr < MEMORY_STRESS_ITERATIONS) { + /* reminder: mem_ctr == 0 is MEMORY_STRESS_INITIAL allocation */ + mem_ctr++; + Serial.print(F("Memory stress increment: ")); + Serial.print(mem_ctr); + Serial.print(F(". Allocating addition memory (bytes): ")); + Serial.println(MEMORY_STRESS_BLOCK_SIZE); + memory_stress[mem_ctr] = (char*)malloc(MEMORY_STRESS_BLOCK_SIZE); + show_memory(); + } +#endif +} /* Arduino loop repeats */ diff --git a/extra/wolfssl/wolfssl/IDE/ARDUINO/sketches/wolfssl_server/README.md b/extra/wolfssl/wolfssl/IDE/ARDUINO/sketches/wolfssl_server/README.md new file mode 100644 index 00000000..523eb087 --- /dev/null +++ b/extra/wolfssl/wolfssl/IDE/ARDUINO/sketches/wolfssl_server/README.md @@ -0,0 +1,134 @@ +# Arduino Basic TLS Server + +Open the [wolfssl_server.ino](./wolfssl_server.ino) file in the Arduino IDE. + +Other IDE products are also supported, such as: + +- [PlatformIO in VS Code](https://docs.platformio.org/en/latest/frameworks/arduino.html) +- [VisualGDB](https://visualgdb.com/tutorials/arduino/) +- [VisualMicro](https://www.visualmicro.com/) + +For examples on other platforms, see the [IDE directory](https://github.com/wolfssl/wolfssl/tree/master/IDE). +Additional examples can be found on [wolfSSL/wolfssl-examples](https://github.com/wolfSSL/wolfssl-examples/). + +## Connect with an Arduino Sketch + +See the companion [Arduino Sketch Client](../wolfssl_client/wolfssl_client.ino). + +## Connect with Linux Client + +See also the [wolfSSL Example TLS Client](https://github.com/wolfSSL/wolfssl/tree/master/examples/client) +and [wolfSSL Example TLS Server](https://github.com/wolfSSL/wolfssl/tree/master/examples/server). + +Assuming a listening [Arduino Sketch Server](./wolfssl_server.ino) at `192.168.1.38` on port `11111`, +connect with the `client` executable: + +``` +./examples/client/client -h 192.168.1.38 -p 11111 -v 3 +``` + +## wolfSSL Error -308 wolfSSL_connect error state on socket + +When using a wired Ethernet connection, and this error is encountered, simply +press the reset button or power cycle the Arduino before making a connection. + +Here's one possible script to test the server from a command-line client: + +```bash +#!/bin/bash +echo "client log " > client_log.txt +counter=1 +THIS_ERR=0 +while [ $THIS_ERR -eq 0 ]; do + ./examples/client/client -h 192.168.1.38 -p 11111 -v 3 >> client_log.txt + + THIS_ERR=$? + if [ $? -ne 0 ]; then + echo "Failed!" + exit 1 + fi + echo "Iteration $counter" + echo "Iteration $counter" >> client_log.txt + ((counter++)) +done +``` + +Output expected from the `client` command: + +``` +$ ./examples/client/client -h 192.168.1.38 -p 11111 -v 3 +Alternate cert chain used + issuer : /C=US/ST=Montana/L=Bozeman/O=Sawtooth/OU=Consulting/CN=www.wolfssl.com/emailAddress=info@wolfssl.com + subject: /C=US/ST=Montana/L=Bozeman/O=wolfSSL/OU=Support/CN=www.wolfssl.com/emailAddress=info@wolfssl.com + altname = example.com + altname = 127.0.0.1 + serial number:01 +SSL version is TLSv1.2 +SSL cipher suite is ECDHE-RSA-AES128-GCM-SHA256 +SSL curve name is SECP256R1 +--- +Server certificate +-----BEGIN CERTIFICATE----- +MIIE6DCCA9CgAwIBAgIBATANBgkqhkiG9w0BAQsFADCBlDELMAkGA1UEBhMCVVMx +EDAOBgNVBAgMB01vbnRhbmExEDAOBgNVBAcMB0JvemVtYW4xETAPBgNVBAoMCFNh +d3Rvb3RoMRMwEQYDVQQLDApDb25zdWx0aW5nMRgwFgYDVQQDDA93d3cud29sZnNz +bC5jb20xHzAdBgkqhkiG9w0BCQEWEGluZm9Ad29sZnNzbC5jb20wHhcNMjMxMjEz +MjIxOTI4WhcNMjYwOTA4MjIxOTI4WjCBkDELMAkGA1UEBhMCVVMxEDAOBgNVBAgM +B01vbnRhbmExEDAOBgNVBAcMB0JvemVtYW4xEDAOBgNVBAoMB3dvbGZTU0wxEDAO +BgNVBAsMB1N1cHBvcnQxGDAWBgNVBAMMD3d3dy53b2xmc3NsLmNvbTEfMB0GCSqG +SIb3DQEJARYQaW5mb0B3b2xmc3NsLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAMCVCOFXQfJxbbfSRUEnAWXGRa7yvCQwuJXOL07W9hyIvHyf+6hn +f/5cnFF194rKB+c1L4/hvXvAL3yrZKgX/Mpde7rgIeVyLm8uhtiVc9qsG1O5Xz/X +GQ0lT+FjY1GLC2Q/rUO4pRxcNLOuAKBjxfZ/C1loeHOmjBipAm2vwxkBLrgQ48bM +QLRpo0YzaYduxLsXpvPo3a1zvHsvIbX9ZlEMvVSz4W1fHLwjc9EJA4kU0hC5ZMMq +0KGWSrzh1Bpbx6DAwWN4D0Q3MDKWgDIjlaF3uhPSl3PiXSXJag3DOWCktLBpQkIJ +6dgIvDMgs1gip6rrxOHmYYPF0pbf2dBPrdcCAwEAAaOCAUUwggFBMB0GA1UdDgQW +BBSzETLJkpiE4sn40DtuA0LKHw6OPDCB1AYDVR0jBIHMMIHJgBQnjmcRdMMmHT/t +M2OzpNgdMOXo1aGBmqSBlzCBlDELMAkGA1UEBhMCVVMxEDAOBgNVBAgMB01vbnRh +bmExEDAOBgNVBAcMB0JvemVtYW4xETAPBgNVBAoMCFNhd3Rvb3RoMRMwEQYDVQQL +DApDb25zdWx0aW5nMRgwFgYDVQQDDA93d3cud29sZnNzbC5jb20xHzAdBgkqhkiG +9w0BCQEWEGluZm9Ad29sZnNzbC5jb22CFDNEGqhsAez2YPJwUQpM0RT6vOlEMAwG +A1UdEwQFMAMBAf8wHAYDVR0RBBUwE4ILZXhhbXBsZS5jb22HBH8AAAEwHQYDVR0l +BBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMA0GCSqGSIb3DQEBCwUAA4IBAQBK/7nl +hZvaU2Z/ByK/thnqQuukEQdi/zlfMzc6hyZxPROyyrhkOHuKmUgOpaRrsZlu4EZR +vRlSrbymfip6fCOnzNteQ31rBMi33ZWt8JGAWcUZkSYnkbhIHOtVtqp9pDjxA7xs +i6qU1jwFepbFBvEmFC51+93lNbMBLLOtYlohmgi+Vvz5okKHhuWpxZnPrhS+4LkI +JA0dXNYU4UyfQLOp6S1Si0y/rEQxZ8GNBoXsD+SZ10t7IQZm1OT1nf+O8IY5WB2k +W+Jj73zJGIeoAiUQPoco+fXvR56lgAgRkGj+0aOoUbk3/9XKfId/a7wsEsjFhYv8 +DMa5hrjJBMNRN9JP +-----END CERTIFICATE----- +Session timeout set to 500 seconds +Client Random : 56A0BB9647B064D3F20947032B74B31FDB4C93DBAC9460BA8AEA213A2B2DD4A8 +SSL-Session: + Protocol : TLSv1.2 + Cipher : TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + Session-ID: 3255404E997FA9C27ECB4F1A20A70E722E4AA504B63A945FC175434D1907EC31 + Session-ID-ctx: + Master-Key: 67F22168BBADD678643BBA76B398277270C29788AC18FD05B57F6B715F49A7BCEEF75BEAF7FE266B0CC058534AF76C1F + TLS session ticket: NONE + Start Time: 1705533296 + Timeout : 500 (sec) + Extended master secret: no +I hear you fa shizzle! +``` + +### Troubleshooting + +When encountering odd errors such as `undefined reference to ``_impure_ptr'`, such as this: + +```text +c:/users/gojimmypi/appdata/local/arduino15/packages/esp32/tools/xtensa-esp32-elf-gcc/esp-2021r2-patch5-8.4.0/bin/../lib/gcc/xtensa-esp32-elf/8.4.0/../../../../xtensa-esp32-elf/bin/ld.exe: C:\Users\gojimmypi\AppData\Local\Temp\arduino\sketches\EAB8D79A02D1ECF107884802D893914E\libraries\wolfSSL\wolfcrypt\src\logging.c.o:(.literal.wolfssl_log+0x8): undefined reference to `_impure_ptr' +collect2.exe: error: ld returned 1 exit status + +exit status 1 + +Compilation error: exit status 1 +``` + +Try cleaning the Arduino cache directories. For Windows, that's typically in: + +```text +C:\Users\%USERNAME%\AppData\Local\Temp\arduino\sketches +``` + +Remove all other boards from other serial ports, leaving one the one being programmed. diff --git a/extra/wolfssl/wolfssl/IDE/ARDUINO/sketches/wolfssl_server/wolfssl_server.ino b/extra/wolfssl/wolfssl/IDE/ARDUINO/sketches/wolfssl_server/wolfssl_server.ino index f331e781..3a894323 100644 --- a/extra/wolfssl/wolfssl/IDE/ARDUINO/sketches/wolfssl_server/wolfssl_server.ino +++ b/extra/wolfssl/wolfssl/IDE/ARDUINO/sketches/wolfssl_server/wolfssl_server.ino @@ -19,161 +19,820 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA */ +/* +Tested with: + +1) Intel Galileo acting as the Client, with a laptop acting as a server using + the server example provided in examples/server. + Legacy Arduino v1.86 was used to compile and program the Galileo + +2) Espressif ESP32 WiFi + +3) Arduino Due, Nano33 IoT, Nano RP-2040 +*/ + +/* + * Note to code editors: the Arduino client and server examples are edited in + * parallel for side-by-side comparison between examples. + */ + +/* If you have a private include, define it here, otherwise edit WiFi params */ +#define MY_PRIVATE_CONFIG "/workspace/my_private_config.h" + +/* set REPEAT_CONNECTION to a non-zero value to continually run the example. */ +#define REPEAT_CONNECTION 1 + +/* Edit this with your other TLS host server address to connect to: */ +/* #define WOLFSSL_TLS_SERVER_HOST "192.168.1.34" */ + +/* wolfssl TLS examples communicate on port 11111 */ +#define WOLFSSL_PORT 11111 + +/* Choose a monitor serial baud rate: 9600, 14400, 19200, 57600, 74880, etc. */ +#define SERIAL_BAUD 115200 + +/* We'll wait up to 2000 milliseconds to properly shut down connection */ +#define SHUTDOWN_DELAY_MS 2000 + +/* Number of times to retry connection. */ +#define RECONNECT_ATTEMPTS 20 + +/* Optional stress test. Define to consume memory until exhausted: */ +/* #define MEMORY_STRESS_TEST */ + +/* Choose client or server example, not both. */ +/* #define WOLFSSL_CLIENT_EXAMPLE */ +#define WOLFSSL_SERVER_EXAMPLE + +#if defined(MY_PRIVATE_CONFIG) + /* the /workspace directory may contain a private config + * excluded from GitHub with items such as WiFi passwords */ + #include MY_PRIVATE_CONFIG + static const char* ssid PROGMEM = MY_ARDUINO_WIFI_SSID; + static const char* password PROGMEM = MY_ARDUINO_WIFI_PASSWORD; +#else + /* when using WiFi capable boards: */ + static const char* ssid PROGMEM = "your_SSID"; + static const char* password PROGMEM = "your_PASSWORD"; +#endif + +#define BROADCAST_ADDRESS "255.255.255.255" + +/* There's an optional 3rd party NTPClient library by Fabrice Weinberg. + * If it is installed, uncomment define USE_NTP_LIB here: */ +/* #define USE_NTP_LIB */ +#ifdef USE_NTP_LIB + #include +#endif #include +/* Important: make sure settings.h appears before any other wolfSSL headers */ +#include +/* Reminder: settings.h includes user_settings.h + * For ALL project wolfSSL settings, see: + * [your path]/Arduino\libraries\wolfSSL\src\user_settings.h */ #include -#include - -#define USE_CERT_BUFFERS_256 #include +#include -#ifdef NO_WOLFSSL_SERVER - #error Please undefine NO_WOLFSSL_SERVER for this example +/* Define DEBUG_WOLFSSL in user_settings.h for more verbose logging. */ +#if defined(DEBUG_WOLFSSL) + #define PROGRESS_DOT F("") +#else + #define PROGRESS_DOT F(".") #endif -const int port = 11111; /* port to listen on */ +/* Convert a macro to a string */ +#define xstr(x) str(x) +#define str(x) #x -int EthernetSend(WOLFSSL* ssl, char* msg, int sz, void* ctx); -int EthernetReceive(WOLFSSL* ssl, char* reply, int sz, void* ctx); +/* optional board-specific networking includes */ +#if defined(ESP32) + #define USING_WIFI + #include + #include + #ifdef USE_NTP_LIB + WiFiUDP ntpUDP; + #endif + /* Ensure the F() flash macro is defined */ + #ifndef F + #define F + #endif + WiFiClient client; + WiFiServer server(WOLFSSL_PORT); +#elif defined(ESP8266) + #define USING_WIFI + #include + WiFiClient client; + WiFiServer server(WOLFSSL_PORT); +#elif defined(ARDUINO_SAM_DUE) + #include + /* There's no WiFi/Ethernet on the Due. Requires Ethernet Shield. + /* Needs "Ethernet by Various" library to be installed. Tested with V2.0.2 */ + #include + EthernetClient client; + EthernetClient server(WOLFSSL_PORT); +#elif defined(ARDUINO_SAMD_NANO_33_IOT) + #define USING_WIFI + #include + #include /* Needs Arduino WiFiNINA library installed manually */ + WiFiClient client; + WiFiServer server(WOLFSSL_PORT); +#elif defined(ARDUINO_ARCH_RP2040) + #define USING_WIFI + #include + #include + WiFiClient client; + WiFiServer server(WOLFSSL_PORT); +#elif defined(USING_WIFI) + #define USING_WIFI + #include + #include + #ifdef USE_NTP_LIB + WiFiUDP ntpUDP; + #endif + WiFiClient client; + WiFiServer server(WOLFSSL_PORT); +/* TODO +#elif defined(OTHER_BOARD) +*/ +#else + #define USING_WIFI + WiFiClient client; + WiFiServer server(WOLFSSL_PORT); +#endif -EthernetServer server(port); -EthernetClient client; +/* Only for syntax highlighters to show interesting options enabled: */ +#if defined(HAVE_SNI) \ + || defined(HAVE_MAX_FRAGMENT) \ + || defined(HAVE_TRUSTED_CA) \ + || defined(HAVE_TRUNCATED_HMAC) \ + || defined(HAVE_CERTIFICATE_STATUS_REQUEST) \ + || defined(HAVE_CERTIFICATE_STATUS_REQUEST_V2) \ + || defined(HAVE_SUPPORTED_CURVES) \ + || defined(HAVE_ALPN) \ + || defined(HAVE_SESSION_TICKET) \ + || defined(HAVE_SECURE_RENEGOTIATION) \ + || defined(HAVE_SERVER_RENEGOTIATION_INFO) +#endif -WOLFSSL_CTX* ctx = NULL; -WOLFSSL* ssl = NULL; -void setup() { - int err; - WOLFSSL_METHOD* method; +/* we expect our IP address from DHCP */ - Serial.begin(9600); +static WOLFSSL_CTX* ctx = NULL; +static WOLFSSL* ssl = NULL; +static char* wc_error_message = (char*)malloc(80 + 1); +static char errBuf[80]; - method = wolfTLSv1_2_server_method(); - if (method == NULL) { - Serial.println("unable to get method"); - return; - } - ctx = wolfSSL_CTX_new(method); - if (ctx == NULL) { - Serial.println("unable to get ctx"); - return; - } - - /* initialize wolfSSL using callback functions */ - wolfSSL_CTX_set_verify(ctx, SSL_VERIFY_NONE, 0); - wolfSSL_SetIOSend(ctx, EthernetSend); - wolfSSL_SetIORecv(ctx, EthernetReceive); - - /* setup the private key and certificate */ - err = wolfSSL_CTX_use_PrivateKey_buffer(ctx, ecc_key_der_256, - sizeof_ecc_key_der_256, WOLFSSL_FILETYPE_ASN1); - if (err != WOLFSSL_SUCCESS) { - Serial.println("error setting key"); - return; - } - err = wolfSSL_CTX_use_certificate_buffer(ctx, serv_ecc_der_256, - sizeof_serv_ecc_der_256, WOLFSSL_FILETYPE_ASN1); - if (err != WOLFSSL_SUCCESS) { - Serial.println("error setting certificate"); - return; - } +#if defined(MEMORY_STRESS_TEST) + #define MEMORY_STRESS_ITERATIONS 100 + #define MEMORY_STRESS_BLOCK_SIZE 1024 + #define MEMORY_STRESS_INITIAL (4*1024) + static char* memory_stress[MEMORY_STRESS_ITERATIONS]; /* typically 1K per item */ + static int mem_ctr = 0; +#endif - /* Start the server */ - server.begin(); - - return; +static int EthernetSend(WOLFSSL* ssl, char* msg, int sz, void* ctx); +static int EthernetReceive(WOLFSSL* ssl, char* reply, int sz, void* ctx); +static int reconnect = RECONNECT_ATTEMPTS; +static int lng_index PROGMEM = 0; /* 0 = English */ + +#if defined(__arm__) + #include + extern char _end; + extern "C" char *sbrk(int i); + static char *ramstart=(char *)0x20070000; + static char *ramend=(char *)0x20088000; +#endif + +/*****************************************************************************/ +/* fail_wait - in case of unrecoverable error */ +/*****************************************************************************/ +int fail_wait(void) { + show_memory(); + + Serial.println(F("Failed. Halt.")); + while (1) { + delay(1000); + } + return 0; } -int EthernetSend(WOLFSSL* ssl, char* msg, int sz, void* ctx) { - int sent = 0; +/*****************************************************************************/ +/* show_memory() to optionally view during debugging. */ +/*****************************************************************************/ +int show_memory(void) +{ +#if defined(__arm__) + struct mallinfo mi = mallinfo(); + + char *heapend=sbrk(0); + register char * stack_ptr asm("sp"); + #if defined(DEBUG_WOLFSSL_VERBOSE) + Serial.print(" arena="); + Serial.println(mi.arena); + Serial.print(" ordblks="); + Serial.println(mi.ordblks); + Serial.print(" uordblks="); + Serial.println(mi.uordblks); + Serial.print(" fordblks="); + Serial.println(mi.fordblks); + Serial.print(" keepcost="); + Serial.println(mi.keepcost); + #endif + + #if defined(DEBUG_WOLFSSL) || defined(MEMORY_STRESS_TEST) + Serial.print("Estimated free memory: "); + Serial.print(stack_ptr - heapend + mi.fordblks); + Serial.println(F(" bytes")); + #endif + + #if (0) + /* Experimental: not supported on all devices: */ + Serial.print("RAM Start %lx\n", (unsigned long)ramstart); + Serial.print("Data/Bss end %lx\n", (unsigned long)&_end); + Serial.print("Heap End %lx\n", (unsigned long)heapend); + Serial.print("Stack Ptr %lx\n",(unsigned long)stack_ptr); + Serial.print("RAM End %lx\n", (unsigned long)ramend); - sent = client.write((byte*)msg, sz); + Serial.print("Heap RAM Used: ",mi.uordblks); + Serial.print("Program RAM Used ",&_end - ramstart); + Serial.print("Stack RAM Used ",ramend - stack_ptr); - return sent; + Serial.print("Estimated Free RAM: %d\n\n",stack_ptr - heapend + mi.fordblks); + #endif +#else + Serial.println(F("show_memory() not implemented for this platform")); +#endif + return 0; } +/*****************************************************************************/ +/* EthernetSend() to send a message string. */ +/*****************************************************************************/ +int EthernetSend(WOLFSSL* ssl, char* message, int sz, void* ctx) { + int sent = 0; + (void)ssl; + (void)ctx; + + sent = client.write((byte*)message, sz); + return sent; +} + +/*****************************************************************************/ +/* EthernetReceive() to receive a reply string. */ +/*****************************************************************************/ int EthernetReceive(WOLFSSL* ssl, char* reply, int sz, void* ctx) { - int ret = 0; + int ret = 0; + (void)ssl; + (void)ctx; - while (client.available() > 0 && ret < sz) { - reply[ret++] = client.read(); - } + while (client.available() > 0 && ret < sz) { + reply[ret++] = client.read(); + } + return ret; +} + +/*****************************************************************************/ +/* Arduino setup_hardware() */ +/*****************************************************************************/ +int setup_hardware(void) { + int ret = 0; + +#if defined(ARDUINO_SAMD_NANO_33_IOT) + Serial.println(F("Detected known tested and working Arduino Nano 33 IoT")); +#elif defined(ARDUINO_ARCH_RP2040) + Serial.println(F("Detected known tested and working Arduino RP-2040")); +#elif defined(__arm__) && defined(ID_TRNG) && defined(TRNG) + /* need to manually turn on random number generator on Arduino Due, etc. */ + pmc_enable_periph_clk(ID_TRNG); + trng_enable(TRNG); + Serial.println(F("Enabled ARM TRNG")); +#endif - return ret; + show_memory(); + randomSeed(analogRead(0)); + return ret; } -void loop() { - int err = 0; - int input = 0; - char errBuf[80]; - char reply[80]; - int replySz = 0; - const char* cipherName; - - /* Listen for incoming client requests. */ - client = server.available(); - if (!client) { +/*****************************************************************************/ +/* Arduino setup_datetime() */ +/* The device needs to have a valid date within the valid range of certs. */ +/*****************************************************************************/ +int setup_datetime(void) { + int ret = 0; + int ntp_tries = 20; + + /* we need a date in the range of cert expiration */ +#ifdef USE_NTP_LIB + #if defined(ESP32) + NTPClient timeClient(ntpUDP, "pool.ntp.org"); + + timeClient.begin(); + timeClient.update(); + delay(1000); + while (!timeClient.isTimeSet() && (ntp_tries > 0)) { + timeClient.forceUpdate(); + Serial.println(F("Waiting for NTP update")); + delay(2000); + ntp_tries--; + } + if (ntp_tries <= 0) { + Serial.println(F("Warning: gave up waiting on NTP")); + } + Serial.println(timeClient.getFormattedTime()); + Serial.println(timeClient.getEpochTime()); + #endif +#endif + +#if defined(ESP32) + /* see esp32-hal-time.c */ + ntp_tries = 5; + /* Replace "pool.ntp.org" with your preferred NTP server */ + configTime(0, 0, "pool.ntp.org"); + + /* Wait for time to be set */ + while ((time(nullptr) <= 100000) && ntp_tries > 0) { + Serial.println(F("Waiting for time to be set...")); + delay(2000); + ntp_tries--; + } +#endif + + return ret; +} /* setup_datetime */ + +/*****************************************************************************/ +/* Arduino setup_network() */ +/*****************************************************************************/ +int setup_network(void) { + int ret = 0; + +#if defined(USING_WIFI) + int status = WL_IDLE_STATUS; + + /* The ESP8266 & ESP32 support both AP and STA. We'll use STA: */ + #if defined(ESP8266) || defined(ESP32) + WiFi.mode(WIFI_STA); + #else + String fv; + if (WiFi.status() == WL_NO_MODULE) { + Serial.println("Communication with WiFi module failed!"); + /* don't continue if no network */ + while (true) ; + } + + fv = WiFi.firmwareVersion(); + if (fv < WIFI_FIRMWARE_LATEST_VERSION) { + Serial.println("Please upgrade the firmware"); + } + #endif + + Serial.print(F("Connecting to WiFi ")); + Serial.print(ssid); + status = WiFi.begin(ssid, password); + while (status != WL_CONNECTED) { + delay(1000); + Serial.print(F(".")); + Serial.print(status); + status = WiFi.status(); + } + + Serial.println(F(" Connected!")); +#else + /* Newer Ethernet shields have a + * MAC address printed on a sticker on the shield */ + byte mac[] = { 0xDE, 0xAD, 0xBE, 0xEF, 0xFE, 0xED }; + IPAddress ip(192, 168, 1, 42); + IPAddress myDns(192, 168, 1, 1); + Ethernet.init(10); /* Most Arduino shields */ + /* Ethernet.init(5); * MKR ETH Shield */ + /* Ethernet.init(0); * Teensy 2.0 */ + /* Ethernet.init(20); * Teensy++ 2.0 */ + /* Ethernet.init(15); * ESP8266 with Adafruit FeatherWing Ethernet */ + /* Ethernet.init(33); * ESP32 with Adafruit FeatherWing Ethernet */ + Serial.println(F("Initialize Ethernet with DHCP:")); + if (Ethernet.begin(mac) == 0) { + Serial.println(F("Failed to configure Ethernet using DHCP")); + /* Check for Ethernet hardware present */ + if (Ethernet.hardwareStatus() == EthernetNoHardware) { + Serial.println(F("Ethernet shield was not found.")); + while (true) { + delay(1); /* do nothing */ + } + } + if (Ethernet.linkStatus() == LinkOFF) { + Serial.println(F("Ethernet cable is not connected.")); + } + /* try to configure using IP address instead of DHCP : */ + Ethernet.begin(mac, ip, myDns); + } + else { + Serial.print(F(" DHCP assigned IP ")); + Serial.println(Ethernet.localIP()); + } + /* We'll assume the Ethernet connection is ready to go. */ +#endif + + Serial.println(F("********************************************************")); + Serial.print(F(" wolfSSL Example Server IP = ")); +#if defined(USING_WIFI) + Serial.println(WiFi.localIP()); +#else + Serial.println(Ethernet.localIP()); +#endif + /* In server mode, there's no host definition. */ + /* See companion example: wolfssl_client.ino */ + Serial.println(F("********************************************************")); + Serial.println(F("Setup network complete.")); + + return ret; +} + +/*****************************************************************************/ +/* Arduino setup_wolfssl() */ +/*****************************************************************************/ +int setup_wolfssl(void) { + int ret = 0; + WOLFSSL_METHOD* method; + + /* Show a revision of wolfssl user_settings.h file in use when available: */ +#if defined(WOLFSSL_USER_SETTINGS_ID) + Serial.print(F("WOLFSSL_USER_SETTINGS_ID: ")); + Serial.println(F(WOLFSSL_USER_SETTINGS_ID)); +#else + Serial.println(F("No WOLFSSL_USER_SETTINGS_ID found.")); +#endif + +#if defined(NO_WOLFSSL_SERVER) + Serial.println(F("wolfSSL server code disabled to save space.")); +#endif +#if defined(NO_WOLFSSL_CLIENT) + Serial.println(F("wolfSSL client code disabled to save space.")); +#endif + +#if defined(DEBUG_WOLFSSL) + wolfSSL_Debugging_ON(); + Serial.println(F("wolfSSL Debugging is On!")); +#else + Serial.println(F("wolfSSL Debugging is Off! (enable with DEBUG_WOLFSSL)")); +#endif + + /* See ssl.c for TLS cache settings. Larger cache = use more RAM. */ +#if defined(NO_SESSION_CACHE) + Serial.println(F("wolfSSL TLS NO_SESSION_CACHE")); +#elif defined(MICRO_SESSION_CACHEx) + Serial.println(F("wolfSSL TLS MICRO_SESSION_CACHE")); +#elif defined(SMALL_SESSION_CACHE) + Serial.println(F("wolfSSL TLS SMALL_SESSION_CACHE")); +#elif defined(MEDIUM_SESSION_CACHE) + Serial.println(F("wolfSSL TLS MEDIUM_SESSION_CACHE")); +#elif defined(BIG_SESSION_CACHE) + Serial.println(F("wolfSSL TLS BIG_SESSION_CACHE")); +#elif defined(HUGE_SESSION_CACHE) + Serial.println(F("wolfSSL TLS HUGE_SESSION_CACHE")); +#elif defined(HUGE_SESSION_CACHE) + Serial.println(F("wolfSSL TLS HUGE_SESSION_CACHE")); +#else + Serial.println(F("WARNING: Unknown or no TLS session cache setting.")); + /* See wolfssl/src/ssl.c for amount of memory used. + * It is best on embedded devices to choose a TLS session cache size. */ +#endif + + ret = wolfSSL_Init(); + if (ret == WOLFSSL_SUCCESS) { + Serial.println("Successfully called wolfSSL_Init"); + } + else { + Serial.println("ERROR: wolfSSL_Init failed"); + } + + /* See companion server example with wolfSSLv23_server_method here. + * method = wolfSSLv23_client_method()); SSL 3.0 - TLS 1.3. + * method = wolfTLSv1_2_client_method(); only TLS 1.2 + * method = wolfTLSv1_3_client_method(); only TLS 1.3 + * + * see Arduino\libraries\wolfssl\src\user_settings.h */ + + Serial.println("Here we go!"); + + method = wolfSSLv23_server_method(); + if (method == NULL) { + Serial.println(F("unable to get wolfssl server method")); + fail_wait(); + } + ctx = wolfSSL_CTX_new(method); + if (ctx == NULL) { + Serial.println(F("unable to get ctx")); + fail_wait(); + } + + return ret; +} + +/*****************************************************************************/ +/* Arduino setup_certificates() */ +/*****************************************************************************/ +int setup_certificates(void) { + int ret = 0; + + Serial.println(F("Initializing certificates...")); + show_memory(); + + /* Use built-in validation, No verification callback function: */ + wolfSSL_CTX_set_verify(ctx, SSL_VERIFY_NONE, 0); + + /* Certificate */ + Serial.println("Initializing certificates..."); + ret = wolfSSL_CTX_use_certificate_buffer(ctx, + CTX_SERVER_CERT, + CTX_SERVER_CERT_SIZE, + CTX_CA_CERT_TYPE); + if (ret == WOLFSSL_SUCCESS) { + Serial.print("Success: use certificate: "); + Serial.println(xstr(CTX_SERVER_CERT)); + } + else { + Serial.print("Error: wolfSSL_CTX_use_certificate_buffer failed: "); + wc_ErrorString(ret, wc_error_message); + Serial.println(wc_error_message); + fail_wait(); + } + + /* Setup private server key */ + ret = wolfSSL_CTX_use_PrivateKey_buffer(ctx, + CTX_SERVER_KEY, + CTX_SERVER_KEY_SIZE, + CTX_SERVER_KEY_TYPE); + if (ret == WOLFSSL_SUCCESS) { + Serial.print("Success: use private key buffer: "); + Serial.println(xstr(CTX_SERVER_KEY)); + } + else { + Serial.print("Error: wolfSSL_CTX_use_PrivateKey_buffer failed: "); + wc_ErrorString(ret, wc_error_message); + Serial.println(wc_error_message); + fail_wait(); + } + + return ret; +} /* Arduino setup */ + +/*****************************************************************************/ +/*****************************************************************************/ +/* Arduino setup() */ +/*****************************************************************************/ +/*****************************************************************************/ +void setup(void) { + int i = 0; + Serial.begin(SERIAL_BAUD); + while (!Serial && (i < 10)) { + /* wait for serial port to connect. Needed for native USB port only */ + delay(1000); + i++; + } + + Serial.println(F("")); + Serial.println(F("")); + Serial.println(F("wolfSSL TLS Server Example Startup.")); + + /* define DEBUG_WOLFSSL in wolfSSL user_settings.h for diagnostics */ +#if defined(DEBUG_WOLFSSL) + wolfSSL_Debugging_ON(); +#endif + + /* Optionally pre-allocate a large block of memory for testing */ +#if defined(MEMORY_STRESS_TEST) + Serial.println(F("WARNING: Memory Stress Test Active!")); + Serial.print(F("Allocating extra memory: ")); + Serial.print(MEMORY_STRESS_INITIAL); + Serial.println(F(" bytes...")); + memory_stress[mem_ctr] = (char*)malloc(MEMORY_STRESS_INITIAL); + show_memory(); +#endif + + setup_hardware(); + + setup_network(); + + setup_datetime(); + + setup_wolfssl(); + + setup_certificates(); + + /* Initialize wolfSSL using callback functions. */ + wolfSSL_SetIOSend(ctx, EthernetSend); + wolfSSL_SetIORecv(ctx, EthernetReceive); + +#if defined THIS_USER_SETTINGS_VERSION + Serial.print(F("This user_settings.h version:")) + Serial.println(THIS_USER_SETTINGS_VERSION) +#endif + + /* Start the server + * See https://www.arduino.cc/reference/en/libraries/ethernet/server.begin/ + */ + + Serial.println(F("Completed Arduino setup()")); + + server.begin(); + Serial.println("Begin Server... (waiting for remote client to connect)"); + + /* See companion wolfssl_client.ino code */ return; - } +} /* Arduino setup */ + +/*****************************************************************************/ +/* wolfSSL error_check() */ +/*****************************************************************************/ +int error_check(int this_ret, bool halt_on_error, + const __FlashStringHelper* message) { + int ret = 0; + if (this_ret == WOLFSSL_SUCCESS) { + Serial.print(F("Success: ")); + Serial.println(message); + } + else { + Serial.print(F("ERROR: return = ")); + Serial.print(this_ret); + Serial.print(F(": ")); + Serial.println(message); + Serial.println(wc_GetErrorString(this_ret)); + if (halt_on_error) { + fail_wait(); + } + } + show_memory(); - if (client.connected()) { + return ret; +} /* error_check */ - Serial.println("Client connected"); +/*****************************************************************************/ +/* wolfSSL error_check_ssl */ +/* Parameters: */ +/* ssl is the current WOLFSSL object pointer */ +/* halt_on_error set to true to suspend operations for critical error */ +/* message is expected to be a memory-efficient F("") macro string */ +/*****************************************************************************/ +int error_check_ssl(WOLFSSL* ssl, int this_ret, bool halt_on_error, + const __FlashStringHelper* message) { + int err = 0; - ssl = wolfSSL_new(ctx); if (ssl == NULL) { - Serial.println("Unable to allocate SSL object"); - return; - } - - err = wolfSSL_accept(ssl); - if (err != WOLFSSL_SUCCESS) { - err = wolfSSL_get_error(ssl, 0); - wolfSSL_ERR_error_string(err, errBuf); - Serial.print("TLS Accept Error: "); - Serial.println(errBuf); - } - - Serial.print("SSL version is "); - Serial.println(wolfSSL_get_version(ssl)); - - cipherName = wolfSSL_get_cipher(ssl); - Serial.print("SSL cipher suite is "); - Serial.println(cipherName); - - Serial.print("Server Read: "); - /* wait for data */ - while (!client.available()) {} - /* read data */ - while (wolfSSL_pending(ssl)) { - input = wolfSSL_read(ssl, reply, sizeof(reply) - 1); - if (input < 0) { - err = wolfSSL_get_error(ssl, 0); - wolfSSL_ERR_error_string(err, errBuf); - Serial.print("TLS Read Error: "); - Serial.println(errBuf); - break; - } else if (input > 0) { - replySz = input; - reply[input] = '\0'; - Serial.print(reply); - } else { - Serial.println(); - } - } - - /* echo data */ - if ((wolfSSL_write(ssl, reply, replySz)) != replySz) { - err = wolfSSL_get_error(ssl, 0); - wolfSSL_ERR_error_string(err, errBuf); - Serial.print("TLS Write Error: "); - Serial.println(errBuf); - } - - wolfSSL_shutdown(ssl); - wolfSSL_free(ssl); - } - - client.stop(); - Serial.println("Connection complete"); + Serial.println(F("ssl is Null; Unable to allocate SSL object?")); +#ifndef DEBUG_WOLFSSL + Serial.println(F("Define DEBUG_WOLFSSL in user_settings.h for more.")); +#else + Serial.println(F("See wolfssl/wolfcrypt/error-crypt.h for codes.")); +#endif + Serial.print(F("ERROR: ")); + Serial.println(message); + show_memory(); + if (halt_on_error) { + fail_wait(); + } + } + else { + err = wolfSSL_get_error(ssl, this_ret); + if (err == WOLFSSL_SUCCESS) { + Serial.print(F("Success m: ")); + Serial.println(message); + } + else { + if (err < 0) { + wolfSSL_ERR_error_string(err, errBuf); + Serial.print(F("WOLFSSL Error: ")); + Serial.print(err); + Serial.print(F("; ")); + Serial.println(errBuf); + } + else { + Serial.println(F("Success: ssl object.")); + } + } + } + + return err; } + +/*****************************************************************************/ +/*****************************************************************************/ +/* Arduino loop() */ +/*****************************************************************************/ +/*****************************************************************************/ +void loop() { + char errBuf[80] = "(no error"; + char reply[80] = "(no reply)"; + const char msg[] = "I hear you fa shizzle!"; + const char* cipherName; + int input = 0; + int replySz = 0; + int retry_shutdown = SHUTDOWN_DELAY_MS; /* max try, once per millisecond */ + int ret = 0; + IPAddress broadcast_address(255, 255, 255, 255); + + /* Listen for incoming client requests. */ + client = server.available(); + if (client) { + Serial.println("Have Client"); + while (!client.connected()) { + /* wait for the client to actually connect */ + delay(10); + } + Serial.print("Client connected from remote IP: "); + Serial.println(client.remoteIP()); + + ssl = wolfSSL_new(ctx); + if (ssl == NULL) { + Serial.println("Unable to allocate SSL object"); + fail_wait(); + } + + ret = wolfSSL_accept(ssl); + if (ret != WOLFSSL_SUCCESS) { + ret = wolfSSL_get_error(ssl, 0); + wolfSSL_ERR_error_string(ret, errBuf); + Serial.print("TLS Accept Error: "); + Serial.println(errBuf); + } + + cipherName = wolfSSL_get_cipher(ssl); + Serial.print("SSL cipher suite is "); + Serial.println(cipherName); + + Serial.print("Server Read: "); + while (!client.available()) { + /* wait for data */ + } + + /* read data */ + while (wolfSSL_pending(ssl)) { + input = wolfSSL_read(ssl, reply, sizeof(reply) - 1); + if (input < 0) { + ret = wolfSSL_get_error(ssl, 0); + wolfSSL_ERR_error_string(ret, errBuf); + Serial.print("TLS Read Error: "); + Serial.println(errBuf); + break; + } + else if (input > 0) { + replySz = input; + reply[input] = '\0'; + Serial.print(reply); + } + else { + Serial.println(""); + } + } + + /* Write our message into reply buffer to send */ + memset(reply, 0, sizeof(reply)); + memcpy(reply, msg, sizeof(msg)); + replySz = strnlen(reply, sizeof(reply)); + + Serial.println("Sending reply..."); + if ((wolfSSL_write(ssl, reply, replySz)) != replySz) { + ret = wolfSSL_get_error(ssl, 0); + wolfSSL_ERR_error_string(ret, errBuf); + Serial.print("TLS Write Error: "); + Serial.println(errBuf); + } + else { + Serial.println("Reply sent!"); + } + + Serial.println("Shutdown!"); + do { + delay(1); + retry_shutdown--; + ret = wolfSSL_shutdown(ssl); + } while ((ret == WOLFSSL_SHUTDOWN_NOT_DONE) && (retry_shutdown > 0)); + + if (retry_shutdown <= 0) { + /* if wolfSSL_free is called before properly shutting down the + * ssl object, undesired results may occur. */ + Serial.println("Warning! Shutdown did not properly complete."); + } + + wolfSSL_free(ssl); + Serial.println("Connection complete."); + if (REPEAT_CONNECTION) { + Serial.println(); + Serial.println("Waiting for next connection."); + } + else { + client.stop(); + Serial.println("Done!"); + while (1) { + /* wait forever if not repeating */ + delay(100); + } + } + } + else { + /* Serial.println("Client not connected. Trying again..."); */ + } + + delay(100); +} /* Arduino loop repeats */ diff --git a/extra/wolfssl/wolfssl/IDE/ARDUINO/sketches/wolfssl_version/README.md b/extra/wolfssl/wolfssl/IDE/ARDUINO/sketches/wolfssl_version/README.md new file mode 100644 index 00000000..3abfe829 --- /dev/null +++ b/extra/wolfssl/wolfssl/IDE/ARDUINO/sketches/wolfssl_version/README.md @@ -0,0 +1,3 @@ +# Arduino Basic Hello World + +This example simply compiles in wolfSSL and shows the current version number. diff --git a/extra/wolfssl/wolfssl/IDE/ARDUINO/sketches/wolfssl_version/wolfssl_version.ino b/extra/wolfssl/wolfssl/IDE/ARDUINO/sketches/wolfssl_version/wolfssl_version.ino new file mode 100644 index 00000000..b052b46e --- /dev/null +++ b/extra/wolfssl/wolfssl/IDE/ARDUINO/sketches/wolfssl_version/wolfssl_version.ino @@ -0,0 +1,24 @@ +#include +#include +#include + +/* Choose a monitor serial baud rate: 9600, 14400, 19200, 57600, 74880, etc. */ +#define SERIAL_BAUD 115200 + +/* Arduino setup */ +void setup() { + Serial.begin(SERIAL_BAUD); + while (!Serial) { + /* wait for serial port to connect. Needed for native USB port only */ + } + Serial.println(F("")); + Serial.println(F("")); + Serial.println(F("wolfSSL setup complete!")); +} + +/* Arduino main application loop. */ +void loop() { + Serial.print("wolfSSL Version: "); + Serial.println(LIBWOLFSSL_VERSION_STRING); + delay(60000); +} diff --git a/extra/wolfssl/wolfssl/IDE/ARDUINO/wolfssl-arduino.sh b/extra/wolfssl/wolfssl/IDE/ARDUINO/wolfssl-arduino.sh index 107f99b1..d48b4166 100755 --- a/extra/wolfssl/wolfssl/IDE/ARDUINO/wolfssl-arduino.sh +++ b/extra/wolfssl/wolfssl/IDE/ARDUINO/wolfssl-arduino.sh @@ -2,19 +2,124 @@ # this script will reformat the wolfSSL source code to be compatible with # an Arduino project -# run as bash ./wolfssl-arduino.sh +# run as bash ./wolfssl-arduino.sh [INSTALL] [path] +# +# ./wolfssl-arduino.sh +# The default is to install to a local wolfSSL directory (`ROOT_DIR`). +# If successfully built, and the INSTALL option is used, tis directory +# is then moved to the target. +# +# ./wolfssl-arduino.sh INSTALL +# Creates a local wolfSSL directory and then moves it to the ARDUINO_ROOT +# +# ./wolfssl-arduino.sh INSTALL /mnt/c/workspace/Arduino-wolfSSL-$USER +# Updates the Arduino-wolfSSL fork for $USER to refresh versions. +# +# To ensure a pristine build, the directory must not exist. +# +# Reminder there's typically no $USER for GitHub actions, but: +# ROOT_DIR="/mnt/c/Users/$USER/Documents/Arduino/libraries" +# +# The company name is "wolfSSL Inc."; There’s a space, no comma, and a period after "Inc." +# The Arduino library name is "wolfssl" (all lower case) +# The Arduino library directory name is "wolfssl" (all lower case) +# The Arduino library include file is "wolfssl.h" (all lower case) +# The Published wolfSSL Arduino Registry is at https://github.com/wolfSSL/Arduino-wolfSSL.git +# See https://downloads.arduino.cc/libraries/logs/github.com/wolfSSL/Arduino-wolfSSL/ +ROOT_DIR="/wolfssl" + +# The Arduino Version will initially have a suffix appended during fine tuning stage. +WOLFSSL_VERSION_ARUINO_SUFFIX="-Arduino.3" + +# For verbose copy, set CP_CMD="-v", otherwise clear it: CP_CMD="cp" +# Do not set to empty string, as copy will fail with this: CP_CMD="" +# CP_CMD="cp -v " +CP_CMD="cp " + +# Specify the executable shell checker you want to use: +MY_SHELLCHECK="shellcheck" + +# There are special circumstances to publish to GitHub repository. +# Typically: https://github.com/wolfSSL/Arduino-wolfSSL +# +# Unlike a local Arduino library that requires a clean directory, +# we'll allow extra files, overwrites, etc. +# +# Note in all cases, the local IDE/ARDUINO/wolfssl must be empty. +THIS_INSTALL_IS_GITHUB="false" + +# Check if the executable is available in the PATH +if command -v "$MY_SHELLCHECK" >/dev/null 2>&1; then + # Run your command here + shellcheck "$0" || exit 1 +else + echo "$MY_SHELLCHECK is not installed. Please install it if changes to this script have been made." +fi + +if ! [ "$CP_CMD" = "cp " ]; then + if [ "$CP_CMD" = "cp -v" ]; then + echo "Copy verbose mode" + else + echo "ERROR: Copy mode not supported: $CP_CMD" + exit 1 + fi +fi + +# Check environment +if [ -n "$WSL_DISTRO_NAME" ]; then + # we found a non-blank WSL environment distro name + current_path="$(pwd)" + pattern="/mnt/?" + if echo "$current_path" | grep -Eq "^$pattern"; then + # if we are in WSL and shared Windows file system, 'ln' does not work. + ARDUINO_ROOT="/mnt/c/Users/$USER/Documents/Arduino/libraries" + else + ARDUINO_ROOT="$HOME/Arduino/libraries" + fi +fi +echo "The Arduino library root is: $ARDUINO_ROOT" + +if [ $# -gt 0 ]; then + THIS_OPERATION="$1" + if [ "$THIS_OPERATION" = "INSTALL" ]; then + THIS_INSTALL_DIR=$2 + + echo "Install is active." + + if [ "$THIS_INSTALL_DIR" = "" ]; then + if [ -d "$ARDUINO_ROOT$ROOT_DIR" ]; then + echo "Error: the installation directory already exists: $ARDUINO_ROOT$ROOT_DIR" + echo "A new directory needs to be created to ensure there are no stray files" + echo "Please delete or move the directory and try again." + exit 1 + fi + else + echo "Installing to $THIS_INSTALL_DIR" + if [ -d "$THIS_INSTALL_DIR/.git" ];then + echo "Target is a GitHub repository." + THIS_INSTALL_IS_GITHUB="true" + else + echo "Target is NOT a GitHub repository." + fi + fi + else + echo "Error: not a valid operation: $THIS_OPERATION" + exit 1 + fi +fi + -ROOT_DIR="/wolfSSL" ROOT_SRC_DIR="${ROOT_DIR}/src" +EXAMPLES_DIR="${ROOT_DIR}/examples" WOLFSSL_SRC="${ROOT_SRC_DIR}/src" WOLFSSL_HEADERS="${ROOT_SRC_DIR}/wolfssl" WOLFCRYPT_ROOT="${ROOT_SRC_DIR}/wolfcrypt" WOLFCRYPT_SRC="${WOLFCRYPT_ROOT}/src" WOLFCRYPT_HEADERS="${WOLFSSL_HEADERS}/wolfcrypt" OPENSSL_DIR="${WOLFSSL_HEADERS}/openssl" -WOLFSSL_VERSION="5.6.4" -# TOP indicates the file directory comes from the top level of the wolfssl repo + +# TOP indicates the file directory for top level of the wolfssl repository. TOP_DIR="../.." WOLFSSL_SRC_TOP="${TOP_DIR}/src" WOLFSSL_HEADERS_TOP="${TOP_DIR}/wolfssl" @@ -24,122 +129,196 @@ WOLFCRYPT_HEADERS_TOP="${WOLFSSL_HEADERS_TOP}/wolfcrypt" OPENSSL_DIR_TOP="${WOLFSSL_HEADERS_TOP}/openssl" -# TODO: Parse version number WOLFSSL_VERSION=$(grep -i "LIBWOLFSSL_VERSION_STRING" ${TOP_DIR}/wolfssl/version.h | cut -d '"' -f 2) +if [ "$WOLFSSL_VERSION" = "" ]; then + echo "ERROR: Could not find wolfSSL Version in ${TOP_DIR}/wolfssl/version.h" + exit 1 +else + echo "Found wolfSSL version $WOLFSSL_VERSION" + echo "# WOLFSSL_VERSION_ARUINO_SUFFIX $WOLFSSL_VERSION_ARUINO_SUFFIX" +fi +echo "" +THIS_DIR=${PWD##*/} -DIR=${PWD##*/} - -if [ "$DIR" = "ARDUINO" ]; then - if [ ! -d ".${ROOT_DIR}" ]; then - mkdir .${ROOT_DIR} +if [ "$THIS_DIR" = "ARDUINO" ]; then + # mkdir ./wolfssl + if [ -d ".${ROOT_DIR}" ]; then + echo "ERROR: $(realpath ".${ROOT_DIR}") is not empty" + exit 1 + else + echo "Step 01: mkdir .${ROOT_DIR}" + mkdir ."${ROOT_DIR}" fi + + # mkdir ./wolfssl/src if [ ! -d ".${ROOT_SRC_DIR}" ]; then - mkdir .${ROOT_SRC_DIR} + echo "Step 02: mkdir .${ROOT_SRC_DIR}" + mkdir ."${ROOT_SRC_DIR}" fi + # mkdir ./wolfssl/src/wolfssl if [ ! -d ".${WOLFSSL_HEADERS}" ]; then - mkdir .${WOLFSSL_HEADERS} + echo "Step 03: mkdir .${WOLFSSL_HEADERS}" + mkdir ."${WOLFSSL_HEADERS}" fi - cp ${WOLFSSL_HEADERS_TOP}/*.h .${WOLFSSL_HEADERS} + # cp ../../wolfssl/*.h ./wolfssl/src/wolfssl + echo "Step 04: cp ${WOLFSSL_HEADERS_TOP}/*.h .${WOLFSSL_HEADERS}" + $CP_CMD "${WOLFSSL_HEADERS_TOP}"/*.h ."${WOLFSSL_HEADERS}" if [ ! -d ".${WOLFCRYPT_HEADERS}" ]; then - mkdir .${WOLFCRYPT_HEADERS} + # mkdir ./wolfssl/src/wolfssl/wolfcrypt + echo "Step 05: mkdir .${WOLFCRYPT_HEADERS}" + mkdir ."${WOLFCRYPT_HEADERS}" + mkdir ."${WOLFCRYPT_HEADERS}/port" + mkdir ."${WOLFCRYPT_HEADERS}/port/atmel" + mkdir ."${WOLFCRYPT_HEADERS}/port/Espressif" fi - cp ${WOLFCRYPT_HEADERS_TOP}/*.h .${WOLFCRYPT_HEADERS} + + # cp ../../wolfssl/wolfcrypt/*.h ./wolfssl/src/wolfssl/wolfcrypt + echo "Step 06: cp ${WOLFCRYPT_HEADERS_TOP}/*.h .${WOLFCRYPT_HEADERS}" + $CP_CMD "${WOLFCRYPT_HEADERS_TOP}"/*.h ."${WOLFCRYPT_HEADERS}" || exit 1 + $CP_CMD "${WOLFCRYPT_HEADERS_TOP}"/port/atmel/*.h ."${WOLFCRYPT_HEADERS}/port/atmel" || exit 1 + $CP_CMD "${WOLFCRYPT_HEADERS_TOP}"/port/Espressif/*.h ."${WOLFCRYPT_HEADERS}/port/Espressif" || exit 1 # Add in source files to wolfcrypt/src if [ ! -d ".${WOLFCRYPT_ROOT}" ]; then - mkdir .${WOLFCRYPT_ROOT} + # mkdir ./wolfssl/src/wolfcrypt + echo "Step 07: mkdir .${WOLFCRYPT_ROOT}" + mkdir ."${WOLFCRYPT_ROOT}" fi + + # mkdir ./wolfssl/src/wolfcrypt/src if [ ! -d ".${WOLFCRYPT_SRC}" ]; then - mkdir .${WOLFCRYPT_SRC} + echo "Step 08: mkdir .${WOLFCRYPT_SRC}" + mkdir ."${WOLFCRYPT_SRC}" + mkdir ."${WOLFCRYPT_SRC}"/port + mkdir ."${WOLFCRYPT_SRC}"/port/atmel + mkdir ."${WOLFCRYPT_SRC}"/port/Espressif fi - cp ${WOLFCRYPT_SRC_TOP}/*.c .${WOLFCRYPT_SRC} - + + # cp ../../wolfcrypt/src/*.c ./wolfssl/src/wolfcrypt/src + echo "Step 09: cp ${WOLFCRYPT_SRC_TOP}/*.c .${WOLFCRYPT_SRC}" + $CP_CMD -r "${WOLFCRYPT_SRC_TOP}"/*.c ."${WOLFCRYPT_SRC}" || exit 1 + $CP_CMD -r "${WOLFCRYPT_SRC_TOP}"/port/atmel/*.c ."${WOLFCRYPT_SRC}"/port/atmel || exit 1 + $CP_CMD -r "${WOLFCRYPT_SRC_TOP}"/port/Espressif/*.c ."${WOLFCRYPT_SRC}"/port/Espressif || exit 1 + # Add in source files to top level src folders if [ ! -d ".${WOLFSSL_SRC}" ]; then - mkdir .${WOLFSSL_SRC} + # mkdir ./wolfssl/src/src + echo "Step 10: mkdir .${WOLFSSL_SRC}" + mkdir ."${WOLFSSL_SRC}" fi - cp ${WOLFSSL_SRC_TOP}/*.c .${WOLFSSL_SRC} + $CP_CMD "${WOLFSSL_SRC_TOP}"/*.c ."${WOLFSSL_SRC}" || exit 1 # put bio and evp as includes - cp .${WOLFSSL_SRC}/bio.c .${WOLFSSL_HEADERS} - cp .${WOLFCRYPT_SRC}/evp.c .${WOLFSSL_HEADERS} + $CP_CMD ."${WOLFSSL_SRC}"/bio.c ."${WOLFSSL_HEADERS}" || exit 1 + $CP_CMD ."${WOLFCRYPT_SRC}"/evp.c ."${WOLFSSL_HEADERS}" || exit 1 # make a copy of evp.c and bio.c for ssl.c to include inline - cp .${WOLFSSL_HEADERS}/evp.c .${WOLFCRYPT_SRC}/evp.c - cp .${WOLFSSL_HEADERS}/bio.c .${WOLFCRYPT_SRC}/bio.c - + $CP_CMD ."${WOLFSSL_HEADERS}"/evp.c ."${WOLFCRYPT_SRC}"/evp.c || exit 1 + $CP_CMD ."${WOLFSSL_HEADERS}"/bio.c ."${WOLFCRYPT_SRC}"/bio.c || exit 1 + # copy openssl compatibility headers to their appropriate location if [ ! -d ".${OPENSSL_DIR}" ]; then - mkdir .${OPENSSL_DIR} + mkdir ."${OPENSSL_DIR}" fi - cp ${OPENSSL_DIR_TOP}/* .${OPENSSL_DIR} + $CP_CMD "${OPENSSL_DIR_TOP}"/* ."${OPENSSL_DIR}" || exit 1 + # Finally, copy the Arduino-specific wolfssl library files into place: [lib]/src + $CP_CMD ./wolfssl.h ".${ROOT_SRC_DIR}"/wolfssl.h - cat > .${ROOT_SRC_DIR}/wolfssl.h < -#include -#include -EOF + echo "Copy examples...." + # Copy examples + mkdir -p ".${ROOT_SRC_DIR}"/examples + echo "Copy wolfssl_client example...." + mkdir -p ".${EXAMPLES_DIR}"/wolfssl_client + $CP_CMD ./sketches/wolfssl_client/wolfssl_client.ino ".${EXAMPLES_DIR}"/wolfssl_client/wolfssl_client.ino || exit 1 + $CP_CMD ./sketches/wolfssl_client/README.md ".${EXAMPLES_DIR}"/wolfssl_client/README.md || exit 1 -# Creates user_settings file if one does not exist - if [ ! -f ".${ROOT_SRC_DIR}/user_settings.h" ]; then - cat > .${ROOT_SRC_DIR}/user_settings.h < "$PREPEND_FILE.tmp" +cat "$PREPEND_FILE.tmp" ${TOP_DIR}/README.md > PREPENDED_README.md -/* RNG DEFAULT !!FOR TESTING ONLY!! */ -/* comment out the error below to get started w/ bad entropy source - * This will need fixed before distribution but is OK to test with */ -#error "needs solved, see: https://www.wolfssl.com/docs/porting-guide/" -#define WOLFSSL_GENSEED_FORTEST +# Here we'll insert the wolfSSL version into the `library.properties.tmp` file, along with an Arduino version suffix. +# The result should be something like version=5.6.6.Arduino.1 (for the 1st incremental version on top of 5.6.6) +sed s/"$VERSION_PLACEHOLDER"/"$WOLFSSL_VERSION"/ "$PROPERTIES_FILE_TEMPLATE" > "library.properties.tmp" +sed -i.backup s/"$ARDUINO_VERSION_SUFFIX_PLACEHOLDER"/"$WOLFSSL_VERSION_ARUINO_SUFFIX"/ "library.properties.tmp" -#endif /* ARDUINO_USER_SETTINGS_H */ -EOF - fi +# cat library.properties.tmp +# echo "${WOLFSSL_VERSION_ARUINO_SUFFIX}" - cp .${WOLFCRYPT_HEADERS}/settings.h .${WOLFCRYPT_HEADERS}/settings.h.bak - cat > .${WOLFCRYPT_HEADERS}/settings.h <> .${WOLFCRYPT_HEADERS}/settings.h - - #Creating library.properties file based off of: - #https://arduino.github.io/arduino-cli/0.35/library-specification/#libraryproperties-file-format - - cat > .${ROOT_DIR}/library.properties < -sentence=A lightweight SSL/TLS library written in ANSI C and targeted for embedded, RTOS, and resource-constrained environments. -paragraph=Manual: https://www.wolfssl.com/documentation/manuals/wolfssl/index.html. -category=Communication -url=https://www.wolfssl.com/ -architectures=* - -EOF +echo "Step 11: Final root file copy" +$CP_CMD PREPENDED_README.md ."${ROOT_DIR}"/README.md || exit 1 +$CP_CMD library.properties.tmp ."${ROOT_DIR}"/library.properties || exit 1 +$CP_CMD "${TOP_DIR}"/"LICENSING" ."${ROOT_DIR}"/ || exit 1 +$CP_CMD "${TOP_DIR}"/"README" ."${ROOT_DIR}"/ || exit 1 +$CP_CMD "${TOP_DIR}"/"COPYING" ."${ROOT_DIR}"/ || exit 1 +$CP_CMD "${TOP_DIR}"/"ChangeLog.md" ."${ROOT_DIR}"/ || exit 1 +$CP_CMD "${TOP_DIR}"/".editorconfig" ."${ROOT_DIR}"/ || exit 1 +$CP_CMD "${TOP_DIR}"/".gitignore" ."${ROOT_DIR}"/ || exit 1 -else - echo "ERROR: You must be in the IDE/ARDUINO directory to run this script" +$CP_CMD "keywords.txt" ."${ROOT_DIR}"/ || exit 1 + + +echo "Step 12: Workspace to publish:" +echo "" +head -n 3 PREPENDED_README.md +echo "" +ls ./wolfssl -al +echo "" + +# Optionally install to a separate directory. +# Note we should have exited above if a problem was encountered, +# as we'll never want to install a bad library. +if [ "$THIS_OPERATION" = "INSTALL" ]; then + if [ "$THIS_INSTALL_IS_GITHUB" = "true" ]; then + echo "Installing to GitHub directory: $THIS_INSTALL_DIR" + cp -r ."$ROOT_DIR"/* "$THIS_INSTALL_DIR" || exit 1 + else + echo "Config:" + echo "cp ../../examples/configs/user_settings_arduino.h ".${ROOT_SRC_DIR}"/user_settings.h" + # Nearly an ordinary copy, but we remove any lines with ">>" (typically edit with caution warning in comments) + grep -v '>>' ../../examples/configs/user_settings_arduino.h > ".${ROOT_SRC_DIR}"/user_settings.h || exit 1 + + # Show the user_settings.h revision string: + grep "WOLFSSL_USER_SETTINGS_ID" ."${ROOT_SRC_DIR}/user_settings.h" + echo "" + + echo "Install:" + echo "mv .$ROOT_DIR $ARDUINO_ROOT" + mv ."$ROOT_DIR" "$ARDUINO_ROOT" || exit 1 + + echo "Arduino wolfSSL Version: $WOLFSSL_VERSION$WOLFSSL_VERSION_ARUINO_SUFFIX" + fi fi + +echo "Done!" diff --git a/extra/wolfssl/wolfssl/IDE/ARDUINO/wolfssl.h b/extra/wolfssl/wolfssl/IDE/ARDUINO/wolfssl.h new file mode 100644 index 00000000..46ef50d3 --- /dev/null +++ b/extra/wolfssl/wolfssl/IDE/ARDUINO/wolfssl.h @@ -0,0 +1,39 @@ +/* wolfssl.h + * + * Copyright (C) 2006-2024 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +/* Edit with caution. This is an Arduino-library specific header for wolfSSL */ + +#ifndef WOLFSSL_USER_SETTINGS + #define WOLFSSL_USER_SETTINGS +#endif + +#include + +/* wolfSSL user_settings.h must be included from settings.h */ +#include +#include + +int wolfSSL_Arduino_Serial_Print(const char *const s) +{ + /* See wolfssl/wolfcrypt/logging.c */ + Serial.println(F(s)); + return 0; +}; diff --git a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/template/components/wolfssl/CMakeLists.txt b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/template/components/wolfssl/CMakeLists.txt index e82e19b6..e129a64e 100644 --- a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/template/components/wolfssl/CMakeLists.txt +++ b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/template/components/wolfssl/CMakeLists.txt @@ -82,8 +82,10 @@ function(FIND_WOLFSSL_DIRECTORY OUTPUT_FOUND_WOLFSSL_DIRECTORY) else() get_filename_component(CURRENT_SEARCH_DIR "$ENV{WOLFSSL_ROOT}" ABSOLUTE) IS_WOLFSSL_SOURCE("${CURRENT_SEARCH_DIR}" FOUND_WOLFSSL) - if("${FOUND_WOLFSSL}") - message(STATUS "Found WOLFSSL_ROOT via Environment Variable:") + if( FOUND_WOLFSSL ) + message(STATUS "Found WOLFSSL_ROOT via Environment Variable: ${CURRENT_SEARCH_DIR}") + set(${OUTPUT_FOUND_WOLFSSL_DIRECTORY} ${CURRENT_SEARCH_DIR} PARENT_SCOPE) + return() else() message(FATAL_ERROR "WOLFSSL_ROOT Environment Variable defined, but path not found:") message(STATUS "$ENV{WOLFSSL_ROOT}") diff --git a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_benchmark/components/wolfssl/CMakeLists.txt b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_benchmark/components/wolfssl/CMakeLists.txt index e82e19b6..e129a64e 100644 --- a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_benchmark/components/wolfssl/CMakeLists.txt +++ b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_benchmark/components/wolfssl/CMakeLists.txt @@ -82,8 +82,10 @@ function(FIND_WOLFSSL_DIRECTORY OUTPUT_FOUND_WOLFSSL_DIRECTORY) else() get_filename_component(CURRENT_SEARCH_DIR "$ENV{WOLFSSL_ROOT}" ABSOLUTE) IS_WOLFSSL_SOURCE("${CURRENT_SEARCH_DIR}" FOUND_WOLFSSL) - if("${FOUND_WOLFSSL}") - message(STATUS "Found WOLFSSL_ROOT via Environment Variable:") + if( FOUND_WOLFSSL ) + message(STATUS "Found WOLFSSL_ROOT via Environment Variable: ${CURRENT_SEARCH_DIR}") + set(${OUTPUT_FOUND_WOLFSSL_DIRECTORY} ${CURRENT_SEARCH_DIR} PARENT_SCOPE) + return() else() message(FATAL_ERROR "WOLFSSL_ROOT Environment Variable defined, but path not found:") message(STATUS "$ENV{WOLFSSL_ROOT}") diff --git a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_benchmark/main/main.c b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_benchmark/main/main.c index 855105e7..5a40d9cf 100644 --- a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_benchmark/main/main.c +++ b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_benchmark/main/main.c @@ -24,21 +24,25 @@ #include "sdkconfig.h" /* wolfSSL */ +/* The wolfSSL user_settings.h file is automatically included by the settings.h + * file and should never be explicitly included in any other source files. + * The settings.h should also be listed above wolfssl library include files. */ #include -#include #include -#include "wolfssl/wolfcrypt/port/Espressif/esp32-crypt.h" +#include #ifndef WOLFSSL_ESPIDF - #warning "Problem with wolfSSL user_settings." - #warning "Check components/wolfssl/include" + #error "Problem with wolfSSL user_settings. " \ + "Check components/wolfssl/include " \ + "and confirm WOLFSSL_USER_SETTINGS is defined, " \ + "typically in the component CMakeLists.txt" #endif #include #include /* set to 0 for one benchmark, -** set to 1 for continous benchmark loop */ -#define BENCHMARK_LOOP 1 +** set to 1 for continuous benchmark loop */ +#define BENCHMARK_LOOP 0 /* check BENCH_ARGV in sdkconfig to determine need to set WOLFSSL_BENCH_ARGV */ #ifdef CONFIG_BENCH_ARGV @@ -129,6 +133,7 @@ void my_atmel_free(int slotId) /* the following are needed by benchmark.c with args */ #ifdef WOLFSSL_BENCH_ARGV char* __argv[WOLFSSL_BENCH_ARGV_MAX_ARGUMENTS]; +#define ARG_BUFF_SIZE 16 int construct_argv() { @@ -137,7 +142,7 @@ int construct_argv() int len = 0; char *_argv; /* buffer for copying the string */ char *ch; /* char pointer to trace the string */ - char buff[16] = { 0 }; /* buffer for a argument copy */ + char buff[ARG_BUFF_SIZE] = { 0 }; /* buffer for a argument copy */ ESP_LOGI(TAG, "construct_argv arg:%s\n", CONFIG_BENCH_ARGV); len = strlen(CONFIG_BENCH_ARGV); @@ -170,7 +175,7 @@ int construct_argv() memset(buff, 0, sizeof(buff)); /* copy each args into buffer */ i = 0; - while ((*ch != ' ') && (*ch != '\0') && (i < 16)) { + while ((*ch != ' ') && (*ch != '\0') && (i <= ARG_BUFF_SIZE)) { buff[i] = *ch; ++i; ++ch; @@ -193,14 +198,15 @@ int construct_argv() void app_main(void) { int stack_start = 0; - ESP_LOGI(TAG, "---------------- wolfSSL Benchmark Example ------------"); + + ESP_LOGI(TAG, "---------------- wolfSSL Benchmark Example -------------"); ESP_LOGI(TAG, "--------------------------------------------------------"); ESP_LOGI(TAG, "--------------------------------------------------------"); ESP_LOGI(TAG, "---------------------- BEGIN MAIN ----------------------"); ESP_LOGI(TAG, "--------------------------------------------------------"); ESP_LOGI(TAG, "--------------------------------------------------------"); -#ifdef HAVE_VERSION_EXTENDED_INFO +#if defined(HAVE_VERSION_EXTENDED_INFO) && defined(WOLFSSL_HAS_METRICS) esp_ShowExtendedSystemInfo(); #endif @@ -236,7 +242,7 @@ void app_main(void) ESP_LOGI(TAG, "Stack used: %d\n", stack_start - uxTaskGetStackHighWaterMark(NULL)); - #ifdef WOLFSSL_HW_METRICS_DISABLED/* Remove _DISABLED upon #6990 Merge */ + #if defined(WOLFSSL_HW_METRICS) && defined(WOLFSSL_HAS_METRICS) esp_hw_show_metrics(); #endif } while (BENCHMARK_LOOP); @@ -249,8 +255,9 @@ void app_main(void) ESP_LOGI(TAG, "Stack HWM: %d\n", uxTaskGetStackHighWaterMark(NULL)); #endif - ESP_LOGI(TAG, "\n\nDone!\n\n" - "If running from idf.py monitor, press twice: Ctrl+]"); +#ifdef WOLFSSL_ESPIDF_EXIT_MESSAGE + ESP_LOGI(TAG, WOLFSSL_ESPIDF_EXIT_MESSAGE); +#endif /* after the test, we'll just wait */ while (1) { diff --git a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/Makefile b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/Makefile index f3854ff1..45d4b1d2 100644 --- a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/Makefile +++ b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/Makefile @@ -1,12 +1,134 @@ +# ESP8266 Project Makefile for wolfssl_client # -# This is a project Makefile. It is assumed the directory this Makefile resides in is a -# project subdirectory. +# Copyright (C) 2006-2024 wolfSSL Inc. +# +# This file is part of wolfSSL. +# +# wolfSSL is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# wolfSSL is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA # +# +# This is a project Makefile. +# It is assumed the directory this Makefile resides in is a +# project subdirectory containing an entire project. +# +# Optional private config headers. Define environment variables +# to include various default header files that are typically +# not in a git path, and thus excluded from being checked in. +# +# Environment Variable Name | Header file name included +# ---------------------------------- | --------------------------------------- +# MY_PRIVATE_CONFIG (files detected / selected in header) +# USE_MY_PRIVATE_WSL_CONFIG /mnt/c/workspace/my_private_config.h +# USE_MY_PRIVATE_MAC_CONFIG ~/Documents/my_private_config.h +# USE_MY_PRIVATE_LINUX_CONFIG ~/workspace/my_private_config.h +# USE_MY_PRIVATE_WINDOWS_CONFIG /workspace/my_private_config.h +# +# PROJECT_NAME := wolfssl_client +MY_PRIVATE_CONFIG ?= n +USE_MY_PRIVATE_WSL_CONFIG ?= n +USE_MY_PRIVATE_MAC_CONFIG ?= n +USE_MY_PRIVATE_LINUX_CONFIG ?= n +USE_MY_PRIVATE_WINDOWS_CONFIG ?= n + +# Calling shell causes unintuitive error in Windows: +# OS := $(shell uname -s) +# +# But OS, or MY_PRIVATE_CONFIG should already be defined: +$(info ************* wolfssl_client *************) + +ifeq ($(MY_PRIVATE_CONFIG),y) + CFLAGS += -DMY_PRIVATE_CONFIG + $(info Enabled MY_PRIVATE_CONFIG") +endif + +# Check for Windows environment variable: USE_MY_PRIVATE_WINDOWS_CONFIG +ifeq ($(USE_MY_PRIVATE_WINDOWS_CONFIG),y) + # This hard coded MY_CONFIG_FILE value must match that in the header file. + MY_CONFIG_FILE := /workspace/my_private_config.h + ifeq ($(wildcard $(MY_CONFIG_FILE)),) + $(info File does not exist: $(MY_CONFIG_FILE)) + else + CFLAGS += -DUSE_MY_PRIVATE_WINDOWS_CONFIG + $(info Using private config file for: Windows) + endif +endif + +# Check for WSL environment variable: USE_MY_PRIVATE_WSL_CONFIG +ifeq ($(USE_MY_PRIVATE_WSL_CONFIG),y) + # This hard coded MY_CONFIG_FILE value must match that in the header file. + MY_CONFIG_FILE := /mnt/c/workspace/my_private_config.h + ifeq ($(wildcard $(MY_CONFIG_FILE)),) + $(info File does not exist: $(MY_CONFIG_FILE)) + else + CFLAGS += -DUSE_MY_PRIVATE_WSL_CONFIG + $(info Using private config file for: WSL) + endif +endif + +# Check for Linux environment variable: USE_MY_PRIVATE_LINUX_CONFIG +ifeq ($(USE_MY_PRIVATE_LINUX_CONFIG),y) + # This hard coded MY_CONFIG_FILE value must match that in the header file. + MY_CONFIG_FILE := ~/workspace/my_private_config.h + ifeq ($(wildcard $(MY_CONFIG_FILE)),) + $(info File does not exist: $(MY_CONFIG_FILE)) + else + CFLAGS += -DUSE_MY_PRIVATE_LINUX_CONFIG + $(info Using private config file for: Linux) + endif +endif + +# Check for Mac environment variable: USE_MY_PRIVATE_MAC_CONFIG +ifeq ($(USE_MY_PRIVATE_MAC_CONFIG),y) + # This hard coded MY_CONFIG_FILE value must match that in the header file. + MY_CONFIG_FILE := ~/Documents/my_private_config.h + ifeq ($(wildcard $(MY_CONFIG_FILE)),) + $(info File does not exist: $(MY_CONFIG_FILE)) + else + CFLAGS += -DUSE_MY_PRIVATE_MAC_CONFIG + $(info Using private config file for: Mac) + endif +endif + +ifneq ($(OS),MY_PRIVATE_CONFIG) + CFLAGS += -DMY_PRIVATE_CONFIG="$(MY_PRIVATE_CONFIG)" +else + ifeq ($(OS),Linux) + CFLAGS += -DOS_LINUX + endif + ifeq ($(OS),Windows_NT) + CFLAGS += -DWOLFSSL_MAKE_SYSTEM_NAME_WINDOWS + endif + ifeq ($(OS),Darwin) + CFLAGS += -DWOLFSSL_MAKE_SYSTEM_NAME_APPLE + endif + ifneq (,$(findstring MINGW,$(OS))) + CFLAGS += -DWOLFSSL_MAKE_SYSTEM_NAME_MINGW + endif + ifneq (,$(findstring CYGWIN,$(OS))) + CFLAGS += -DWOLFSSL_MAKE_SYSTEM_NAME_CYGWIN + endif +endif + +# It is essential that the build process sees the WOLFSSL_USER_SETTINGS CFLAGS += -DWOLFSSL_USER_SETTINGS -# if there isn't the directory, please disable the line below. + +# if directory not available, please disable the line below. EXTRA_COMPONENT_DIRS = $(IDF_PATH)/examples/common_components/protocol_examples_common +# The Standard Espressif IDF include: include $(IDF_PATH)/make/project.mk diff --git a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/README.md b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/README.md index 21855add..1bfd0cc8 100644 --- a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/README.md +++ b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/README.md @@ -8,12 +8,50 @@ When using the CLI, see the [example parameters](/IDE/Espressif/ESP-IDF/examples For general information on [wolfSSL examples for Espressif](../README.md), see the [README](https://github.com/wolfSSL/wolfssl/blob/master/IDE/Espressif/ESP-IDF/README.md) file. -## VisualGDB +## Quick Start + +Use the [ESP-IDF](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/get-started/index.html) +for ESP32 or [RTOS SDK](https://docs.espressif.com/projects/esp8266-rtos-sdk/en/latest/get-started/index.html) +for the ESP8266. + +Run `menuconfig` utility (`idf.py menuconfig` for ESP32 or `make menuconfig` for the ESP8266) +and set the various parameters for the target device, along with local WiFi settings: + +* Target Host: `CONFIG_WOLFSSL_TARGET_HOST` (The IP address of a listening server) +* Target Port: `CONFIG_WOLFSSL_TARGET_PORT` (Typically `11111`) +* Example WiFi SSID: `CONFIG_EXAMPLE_WIFI_SSID` (The WiFi that you want to connect to) +* Example WiFi Password: `CONFIG_EXAMPLE_WIFI_PASSWORD` (The WiFi password) + +The latest examples use makefiles that do not require local file copy installation of wolfSSL. + +Build and flash the software to see the example in action. + +## Quick Start with VisualGDB + +There are optional [VisualGDB](https://visualgdb.com/tutorials/esp8266/) project files in the +[VisualGDB](./VisualGDB) project subdirectory, and an ESP8266 project file in the project directory, +called `wolfssl_client_ESP8266.vgdbproj`. Open the VisualGDB Visual Studio Project file in the VisualGDB directory and click the "Start" button. -No wolfSSL setup is needed. You may need to adjust your specific COM port. The default is `COM20`. +No wolfSSL setup is needed. You may need to adjust your specific COM port. The default is `COM19`. + +## Troubleshooting + +Weird results, odd messages, unexpected compiler errors? Manually delete the build directory and +any locally generated files (`sdkconfig`, `sdkconfig-debug`, etc.) and start over. + +The `build` directory is typically located in the root of the project directory: `[project]/build`. + + +Difficulty flashing: + +* Ensure the target device has a robust, stable, clean power supply. +* Check that quality USB cables are being used. +* Try lowering the flash baud rate in the `menuconfig`. The 115200 is typically reliable. +* Review board specifications: some require manual boot mode via on-board buttons. +* See [Espressif ESP Frequently Asked Questions](https://docs.espressif.com/projects/esp-faq/en/latest/esp-faq-en-master.pdf) -## ESP-IDF Commandline +## ESP-IDF Commandline v5.x 1. `idf.py menuconfig` to config the project @@ -38,9 +76,83 @@ When you want to test the wolfSSL client e.g. Launch ./examples/server/server -v 4 -b -i -d + +## VisualGDB for ESP8266 + +Reminder that we build with `make` and not `cmake` in VisualGDB. + +Build files will be created in `[project directory]\build` + +## ESP-IDF make Commandline (version 3.5 or earlier for the ESP8266) + +``` +export IDF_PATH=~/esp/ESP8266_RTOS_SDK + +``` + + +## ESP-IDF CMake Commandline (version 3.5 or earlier for the ESP8266) + +Build files will be created in `[project directory]\build\debug` + +``` +# Set your path to RTOS SDK, shown here for default from WSL with VisualGDB +WRK_IDF_PATH=/mnt/c/SysGCC/esp8266/rtos-sdk/v3.4 +# or +WRK_IDF_PATH=~/esp/ESP8266_RTOS_SDK + +# Setup the environment +. $WRK_IDF_PATH/export.sh + +# install as needed / prompted +/mnt/c/SysGCC/esp8266/rtos-sdk/v3.4/install.sh + +# Fetch wolfssl from GitHub if needed: +cd /workspace +git clone https://github.com/wolfSSL/wolfssl.git + +# change directory to wolfssl client example. +cd wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client + +# or for example, WSL with C:\workspace as home for git clones: +# cd /mnt/c/workspace/wolfssl-$USER/IDE/Espressif/ESP-IDF/examples/wolfssl_client + +# adjust settings as desired +idf.py menuconfig + + +idf.py build flash -p /dev/ttyS70 -b 115200 +idf.py monitor -p /dev/ttyS70 -b 74880 +``` + ## SM Ciphers -#### Working Linux Client to ESP32 Server +(TODO coming soon) +See https://github.com/wolfSSL/wolfsm + +#### Working Linux Client to ESP32 Server Example: + +``` +./examples/client/client -h 192.168.1.37 -p 11111 -v 3 +``` + +```text +-c Certificate file, default ./certs/client-cert.pem +-k Key file, default ./certs/client-key.pem +-A Certificate Authority file, default ./certs/ca-cert.pem +``` + +Example client, with default certs explicitly given: + +```bash +./examples/client/client -h 192.168.1.37 -p 11111 -v 3 -c ./certs/client-cert.pem -k ./certs/client-key.pem -A ./certs/ca-cert.pem +``` + +Example client, with RSA 1024 certs explicitly given: + +``` +./examples/client/client -h 192.168.1.37 -p 11111 -v 3 -c ./certs/1024/client-cert.pem -k ./certs/1024/client-key.pem -A ./certs/1024/ca-cert.pem +``` Command: @@ -48,7 +160,6 @@ Command: cd /mnt/c/workspace/wolfssl-$USER/IDE/Espressif/ESP-IDF/examples/wolfssl_server . /mnt/c/SysGCC/esp32/esp-idf/v5.1/export.sh idf.py flash -p /dev/ttyS19 -b 115200 monitor - ``` ``` @@ -75,4 +186,3 @@ I hear you fa shizzle! ``` See the README.md file in the upper level 'examples' directory for [more information about examples](../README.md). - diff --git a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/VisualGDB/wolfssl_client_IDF_v5_ESP32.sln b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/VisualGDB/wolfssl_client_IDF_v5_ESP32.sln index a2be094c..e8e3a317 100644 --- a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/VisualGDB/wolfssl_client_IDF_v5_ESP32.sln +++ b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/VisualGDB/wolfssl_client_IDF_v5_ESP32.sln @@ -18,6 +18,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "include", "include", "{5326 EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{A51226B3-88A7-4463-B443-0E321C4A3D53}" ProjectSection(SolutionItems) = preProject + ..\..\..\..\..\..\wolfssl\wolfcrypt\error-crypt.h = ..\..\..\..\..\..\wolfssl\wolfcrypt\error-crypt.h + ..\..\..\..\..\..\wolfssl\error-ssl.h = ..\..\..\..\..\..\wolfssl\error-ssl.h + ..\main\Kconfig.projbuild = ..\main\Kconfig.projbuild + ..\build\VisualGDB\Debug\esp-idf\esp_system\ld\memory.ld = ..\build\VisualGDB\Debug\esp-idf\esp_system\ld\memory.ld ..\..\..\..\..\..\..\my_private_config.h = ..\..\..\..\..\..\..\my_private_config.h ..\partitions_singleapp_large.csv = ..\partitions_singleapp_large.csv ..\README.md = ..\README.md @@ -25,6 +29,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution ..\build\VisualGDB\Debug\config\sdkconfig.cmake = ..\build\VisualGDB\Debug\config\sdkconfig.cmake ..\sdkconfig.defaults = ..\sdkconfig.defaults ..\build\VisualGDB\Debug\config\sdkconfig.h = ..\build\VisualGDB\Debug\config\sdkconfig.h + ..\build\VisualGDB\Debug\esp-idf\esp_system\ld\sections.ld = ..\build\VisualGDB\Debug\esp-idf\esp_system\ld\sections.ld EndProjectSection EndProject Global diff --git a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/components/wolfssl/CMakeLists.txt b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/components/wolfssl/CMakeLists.txt index e82e19b6..615142ba 100644 --- a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/components/wolfssl/CMakeLists.txt +++ b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/components/wolfssl/CMakeLists.txt @@ -82,8 +82,10 @@ function(FIND_WOLFSSL_DIRECTORY OUTPUT_FOUND_WOLFSSL_DIRECTORY) else() get_filename_component(CURRENT_SEARCH_DIR "$ENV{WOLFSSL_ROOT}" ABSOLUTE) IS_WOLFSSL_SOURCE("${CURRENT_SEARCH_DIR}" FOUND_WOLFSSL) - if("${FOUND_WOLFSSL}") - message(STATUS "Found WOLFSSL_ROOT via Environment Variable:") + if( FOUND_WOLFSSL ) + message(STATUS "Found WOLFSSL_ROOT via Environment Variable: ${CURRENT_SEARCH_DIR}") + set(${OUTPUT_FOUND_WOLFSSL_DIRECTORY} ${CURRENT_SEARCH_DIR} PARENT_SCOPE) + return() else() message(FATAL_ERROR "WOLFSSL_ROOT Environment Variable defined, but path not found:") message(STATUS "$ENV{WOLFSSL_ROOT}") @@ -156,8 +158,8 @@ if(CMAKE_BUILD_EARLY_EXPANSION) idf_component_register( REQUIRES "${COMPONENT_REQUIRES}" PRIV_REQUIRES # esp_hw_support - esp_timer - driver # this will typically only be needed for wolfSSL benchmark + # esp_timer + # driver # this will typically only be needed for wolfSSL benchmark ) else() diff --git a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/components/wolfssl/component.mk b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/components/wolfssl/component.mk index 13834b08..8184da11 100644 --- a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/components/wolfssl/component.mk +++ b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/components/wolfssl/component.mk @@ -17,24 +17,228 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA # + # # Component Makefile # +# +# The Espressif Managed Components are only for newer versions of the ESP-IDF +# Typically only for ESP32[-x] targets and only for ESP-IDF v4.3 or later: +# See https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-guides/tools/idf-component-manager.html +# https://components.espressif.com/ +# +# Usage: +# +# make flash +# +# make flash ESPPORT=/dev/ttyS55 +# +# make flash ESPBAUD=9600 +# +# make monitor ESPPORT=COM1 +# +# make monitor ESPPORT=/dev/ttyS55 MONITORBAUD=115200 +# +# export ESPPORT=/dev/ttyS55 +# +# https://docs.espressif.com/projects/esp8266-rtos-sdk/en/latest/get-started/index.html +# -COMPONENT_ADD_INCLUDEDIRS := . ./include +# Although the project should define WOLFSSL_USER_SETTINGS, we'll also +# define it here: +CFLAGS +=-DWOLFSSL_USER_SETTINGS -COMPONENT_ADD_INCLUDEDIRS += "$ENV{IDF_PATH}/components/freertos/include/freertos" -# COMPONENT_ADD_INCLUDEDIRS += "$ENV{IDF_PATH}/soc/esp32s3/include/soc" +# NOTICE: the WOLFSSL_ROOT setting MUST be relative! +# See https://docs.espressif.com/projects/esp8266-rtos-sdk/en/latest/api-guides/build-system.html?highlight=must+relative#optional-component-specific-variables +# In the wolfSSL GitHub examples for Espressif: +# https://github.com/wolfSSL/wolfssl/tree/master/IDE/Espressif/ESP-IDF/examples +# When this wolfssl component.mk makefile is in [project]/components/wolfssl +# The root is 7 directories up from here: +WOLFSSL_ROOT := ../../../../../../.. -COMPONENT_SRCDIRS := src wolfcrypt/src -COMPONENT_SRCDIRS += wolfcrypt/src/port/Espressif -COMPONENT_SRCDIRS += wolfcrypt/src/port/atmel -COMPONENT_SRCDIRS += wolfcrypt/benchmark -COMPONENT_SRCDIRS += wolfcrypt/test +# NOTE: The wolfSSL include diretory (e.g. user_settings.h) is +# located HERE in THIS project, and *not* in the wolfSSL root. +COMPONENT_ADD_INCLUDEDIRS := ./include +COMPONENT_ADD_INCLUDEDIRS += $(WOLFSSL_ROOT)/. +COMPONENT_ADD_INCLUDEDIRS += $(WOLFSSL_ROOT)/wolfssl +COMPONENT_ADD_INCLUDEDIRS += $(WOLFSSL_ROOT)/wolfssl/wolfcrypt +COMPONENT_ADD_INCLUDEDIRS += $(WOLFSSL_ROOT)/wolfssl/wolfcrypt/port/Espressif +# COMPONENT_ADD_INCLUDEDIRS += $ENV(IDF_PATH)/components/freertos/include/freertos +# COMPONENT_ADD_INCLUDEDIRS += "$ENV(IDF_PATH)/soc/esp32s3/include/soc" -CFLAGS +=-DWOLFSSL_USER_SETTINGS +# wolfSSL +COMPONENT_SRCDIRS := $(WOLFSSL_ROOT)/src + +# wolfcrypt +COMPONENT_SRCDIRS += $(WOLFSSL_ROOT)/wolfcrypt/src + +# Espressif +COMPONENT_SRCDIRS += $(WOLFSSL_ROOT)/wolfcrypt/src/port/Espressif +COMPONENT_SRCDIRS += $(WOLFSSL_ROOT)/wolfcrypt/src/port/atmel + +COMPONENT_OBJEXCLUDE := $(WOLFSSL_ROOT)/wolfcrypt/src/aes_asm.o +COMPONENT_OBJEXCLUDE += $(WOLFSSL_ROOT)/wolfcrypt/src/evp.o +COMPONENT_OBJEXCLUDE += $(WOLFSSL_ROOT)/wolfcrypt/src/misc.o +COMPONENT_OBJEXCLUDE += $(WOLFSSL_ROOT)/wolfcrypt/src/sha512_asm.o +COMPONENT_OBJEXCLUDE += $(WOLFSSL_ROOT)/wolfcrypt/src/fe_x25519_asm.o +COMPONENT_OBJEXCLUDE += $(WOLFSSL_ROOT)/wolfcrypt/src/aes_gcm_x86_asm.o +COMPONENT_OBJEXCLUDE += $(WOLFSSL_ROOT)/src/bio.o + + +## +## wolfSSL +## +COMPONENT_OBJS := $(WOLFSSL_ROOT)/src/bio.o +# COMPONENT_OBJS += src/conf.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/src/crl.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/src/dtls.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/src/dtls13.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/src/internal.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/src/keys.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/src/ocsp.o +# COMPONENT_OBJS += src/pk.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/src/quic.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/src/sniffer.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/src/ssl.o +# COMPONENT_OBJS += src/ssl_asn1.o +# COMPONENT_OBJS += src/ssl_bn.o +# COMPONENT_OBJS += src/ssl_certman.o +# COMPONENT_OBJS += src/ssl_crypto.o +# COMPONENT_OBJS += src/ssl_misc.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/src/tls.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/src/tls13.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/src/wolfio.o +# COMPONENT_OBJS += src/x509.o +# COMPONENT_OBJS += src/x509_str.o + +## +## wolfcrypt +## +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/aes.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/arc4.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/asm.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/asn.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/async.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/blake2b.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/blake2s.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/camellia.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/chacha.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/chacha20_poly1305.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/cmac.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/coding.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/compress.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/cpuid.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/cryptocb.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/curve25519.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/curve448.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/des3.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/dh.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/dilithium.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/dsa.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/ecc.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/eccsi.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/ecc_fp.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/ed25519.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/ed448.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/error.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/evp.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/ext_kyber.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/ext_lms.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/ext_xmss.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/falcon.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/fe_448.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/fe_low_mem.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/fe_operations.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/fips.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/fips_test.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/ge_448.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/ge_low_mem.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/ge_operations.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/hash.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/hmac.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/hpke.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/integer.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/kdf.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/logging.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/md2.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/md4.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/md5.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/memory.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/misc.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/pkcs12.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/pkcs7.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/poly1305.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/pwdbased.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/random.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/rc2.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/ripemd.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/rsa.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/sakke.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/selftest.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/sha.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/sha256.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/sha3.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/sha512.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/signature.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/siphash.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/sm2.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/sm3.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/sm4.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/sphincs.o +# COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/sp_arm32.o +# COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/sp_arm64.o +# COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/sp_armthumb.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/sp_c32.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/sp_c64.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/sp_cortexm.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/sp_dsp32.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/sp_int.o +# COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/sp_sm2_arm32.o +# COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/sp_sm2_arm64.o +# COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/sp_sm2_armthumb.o +# COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/sp_sm2_c32.o +# COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/sp_sm2_c64.o +# COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/sp_sm2_cortexm.o +# COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/sp_sm2_x86_64.o +# COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/sp_x86_64.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/srp.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/tfm.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/wc_dsp.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/wc_encrypt.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/wc_kyber.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/wc_kyber_poly.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/wc_lms.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/wc_pkcs11.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/wc_port.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/wc_xmss.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/wolfcrypt_first.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/wolfcrypt_last.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/wolfevent.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/wolfmath.o + +## +## Espressif +## +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/port/Espressif/esp32_aes.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/port/Espressif/esp32_mp.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/port/Espressif/esp32_sha.o +COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/src/port/Espressif/esp32_util.o + +## +## wolfcrypt benchmark (optional) +## +## COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/benchmark/benchmark.o +## COMPONENT_SRCDIRS += $(WOLFSSL_ROOT)/wolfcrypt/benchmark +## COMPONENT_ADD_INCLUDEDIRS += $(WOLFSSL_ROOT)/wolfcrypt/benchmark + + +## +## wolfcrypt test (optional) +## +## COMPONENT_OBJS += $(WOLFSSL_ROOT)/wolfcrypt/test/test.o +## COMPONENT_SRCDIRS += $(WOLFSSL_ROOT)/wolfcrypt/test -COMPONENT_OBJEXCLUDE := wolfcrypt/src/aes_asm.o -COMPONENT_OBJEXCLUDE += wolfcrypt/src/evp.o -COMPONENT_OBJEXCLUDE += wolfcrypt/src/misc.o -COMPONENT_OBJEXCLUDE += src/bio.o +## +## wolfcrypt +## +# COMPONENT_PRIV_INCLUDEDIRS += $(PROJECT_PATH)/components/wolfssl/include diff --git a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/components/wolfssl/include/user_settings.h b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/components/wolfssl/include/user_settings.h index de5e247c..99b61e15 100644 --- a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/components/wolfssl/include/user_settings.h +++ b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/components/wolfssl/include/user_settings.h @@ -1,6 +1,6 @@ /* user_settings.h * - * Copyright (C) 2006-2023 wolfSSL Inc. + * Copyright (C) 2006-2024 wolfSSL Inc. * * This file is part of wolfSSL. * @@ -19,12 +19,16 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA */ +/* Standardized wolfSSL Espressif ESP32 + ESP8266 user_settings.h V5.6.6-01 */ + /* This user_settings.h is for Espressif ESP-IDF */ #include +#define DEBUG_WOLFSSL +#define DEBUG_WOLFSSL_VERBOSE /* The Espressif sdkconfig will have chipset info. ** -** Possible values: +** Some possible values: ** ** CONFIG_IDF_TARGET_ESP32 ** CONFIG_IDF_TARGET_ESP32S2 @@ -37,7 +41,7 @@ #define WOLFSSL_ESPIDF /* - * choose ONE of these Espressif chips to define: + * ONE of these Espressif chipsets should be defined: * * WOLFSSL_ESP32 * WOLFSSL_ESPWROOM32SE @@ -46,8 +50,13 @@ #undef WOLFSSL_ESPWROOM32SE #undef WOLFSSL_ESP8266 #undef WOLFSSL_ESP32 +/* See below for chipset detection from sdkconfig.h */ -#define WOLFSSL_ESP32 +/* Small session cache saves a lot of RAM for ClientCache and SessionCache. + * Memory requirement is about 5KB, otherwise 20K is needed when not specified. + * If extra small footprint is needed, try MICRO_SESSION_CACHE (< 1K) + * When really desparate, try NO_SESSION_CACHE. */ +#define SMALL_SESSION_CACHE /* optionally turn off SHA512/224 SHA512/256 */ /* #define WOLFSSL_NOSHA512_224 */ @@ -62,6 +71,9 @@ #define BENCH_EMBEDDED #define USE_CERT_BUFFERS_2048 +#define WOLFSSL_SMALL_STACK +#define HAVE_ECC +#define RSA_LOW_MEM /* TLS 1.3 */ #define WOLFSSL_TLS13 @@ -79,7 +91,9 @@ #define HAVE_AESGCM -#define WOLFSSL_RIPEMD +/* Optional RIPEMD: RACE Integrity Primitives Evaluation Message Digest */ +/* #define WOLFSSL_RIPEMD */ + /* when you want to use SHA224 */ #define WOLFSSL_SHA224 @@ -87,24 +101,17 @@ #define WOLFSSL_SHA384 /* when you want to use SHA512 */ -#define WOLFSSL_SHA512 +/* #define WOLFSSL_SHA512 */ /* when you want to use SHA3 */ -#define WOLFSSL_SHA3 +/* #define WOLFSSL_SHA3 */ -#define HAVE_ED25519 /* ED25519 requires SHA512 */ +/* ED25519 requires SHA512 */ +/* #define HAVE_ED25519 */ -#define HAVE_ECC -#define HAVE_CURVE25519 -#define CURVE25519_SMALL -#define HAVE_ED25519 - - #define OPENSSL_EXTRA /* when you want to use pkcs7 */ /* #define HAVE_PKCS7 */ -#define HAVE_PKCS7 - #if defined(HAVE_PKCS7) #define HAVE_AES_KEYWRAP #define HAVE_X963_KDF @@ -125,7 +132,7 @@ /* #define CUSTOM_SLOT_ALLOCATION */ #endif -/* rsa primitive specific definition */ +/* RSA primitive specific definition */ #if defined(WOLFSSL_ESP32) || defined(WOLFSSL_ESPWROOM32SE) /* Define USE_FAST_MATH and SMALL_STACK */ #define ESP32_USE_RSA_PRIMITIVE @@ -145,8 +152,6 @@ #endif #endif -#define RSA_LOW_MEM - /* #define WOLFSSL_ATECC508A_DEBUG */ /* date/time */ @@ -173,10 +178,6 @@ /* #undef USE_FAST_MATH */ /* #define USE_INTEGER_HEAP_MATH */ - -#define WOLFSSL_SMALL_STACK - - #define HAVE_VERSION_EXTENDED_INFO /* #define HAVE_WC_INTROSPECTION */ @@ -190,7 +191,6 @@ #define WOLFSSL_CERT_EXT #define WOLFSSL_SYS_CA_CERTS - #define WOLFSSL_CERT_TEXT #define WOLFSSL_ASN_TEMPLATE @@ -203,7 +203,7 @@ #undef WOLFSSL_SYS_CA_CERTS */ -/* +/* command-line options --enable-keygen --enable-certgen --enable-certreq @@ -211,10 +211,11 @@ --enable-asn-template */ -/* Default is HW enabled unless turned off. -** Uncomment these lines to force SW instead of HW acceleration */ - +/* Chipset detection from sdkconfig.h + * Default is HW enabled unless turned off. + * Uncomment lines to force SW instead of HW acceleration */ #if defined(CONFIG_IDF_TARGET_ESP32) + #define WOLFSSL_ESP32 /* wolfSSL HW Acceleration supported on ESP32. Uncomment to disable: */ /* #define NO_ESP32_CRYPT */ /* #define NO_WOLFSSL_ESP32_CRYPT_HASH */ @@ -232,6 +233,7 @@ /***** END CONFIG_IDF_TARGET_ESP32 *****/ #elif defined(CONFIG_IDF_TARGET_ESP32S2) + #define WOLFSSL_ESP32 /* wolfSSL HW Acceleration supported on ESP32-S2. Uncomment to disable: */ /* #define NO_ESP32_CRYPT */ /* #define NO_WOLFSSL_ESP32_CRYPT_HASH */ @@ -244,6 +246,7 @@ /***** END CONFIG_IDF_TARGET_ESP32S2 *****/ #elif defined(CONFIG_IDF_TARGET_ESP32S3) + #define WOLFSSL_ESP32 /* wolfSSL HW Acceleration supported on ESP32-S3. Uncomment to disable: */ /* #define NO_ESP32_CRYPT */ /* #define NO_WOLFSSL_ESP32_CRYPT_HASH */ @@ -257,6 +260,7 @@ #elif defined(CONFIG_IDF_TARGET_ESP32C2) || \ defined(CONFIG_IDF_TARGET_ESP8684) + #define WOLFSSL_ESP32 /* ESP8684 is essentially ESP32-C2 chip + flash embedded together in a * single QFN 4x4 mm package. Out of released documentation, Technical * Reference Manual as well as ESP-IDF Programming Guide is applicable @@ -282,6 +286,7 @@ /***** END CONFIG_IDF_TARGET_ESP32C2 *****/ #elif defined(CONFIG_IDF_TARGET_ESP32C3) + #define WOLFSSL_ESP32 /* wolfSSL HW Acceleration supported on ESP32-C3. Uncomment to disable: */ /* #define NO_ESP32_CRYPT */ @@ -299,6 +304,7 @@ /***** END CONFIG_IDF_TARGET_ESP32C3 *****/ #elif defined(CONFIG_IDF_TARGET_ESP32C6) + #define WOLFSSL_ESP32 /* wolfSSL HW Acceleration supported on ESP32-C6. Uncomment to disable: */ /* #define NO_ESP32_CRYPT */ @@ -315,6 +321,7 @@ /***** END CONFIG_IDF_TARGET_ESP32C6 *****/ #elif defined(CONFIG_IDF_TARGET_ESP32H2) + #define WOLFSSL_ESP32 /* wolfSSL Hardware Acceleration not yet implemented */ #define NO_ESP32_CRYPT #define NO_WOLFSSL_ESP32_CRYPT_HASH @@ -323,15 +330,28 @@ /***** END CONFIG_IDF_TARGET_ESP32H2 *****/ #elif defined(CONFIG_IDF_TARGET_ESP8266) - /* TODO: Revisit ESP8266 */ + #define WOLFSSL_ESP8266 + + /* There's no hardware encryption on the ESP8266 */ + /* Consider using the ESP32-C2/C3/C6 + * See https://www.espressif.com/en/products/socs/esp32-c2 */ #define NO_ESP32_CRYPT #define NO_WOLFSSL_ESP32_CRYPT_HASH #define NO_WOLFSSL_ESP32_CRYPT_AES #define NO_WOLFSSL_ESP32_CRYPT_RSA_PRI /***** END CONFIG_IDF_TARGET_ESP266 *****/ +#elif defined(CONFIG_IDF_TARGET_ESP8684) + /* There's no Hardware Acceleration available on ESP8684 */ + #define NO_ESP32_CRYPT + #define NO_WOLFSSL_ESP32_CRYPT_HASH + #define NO_WOLFSSL_ESP32_CRYPT_AES + #define NO_WOLFSSL_ESP32_CRYPT_RSA_PRI + /***** END CONFIG_IDF_TARGET_ESP8684 *****/ + #else /* Anything else encountered, disable HW accleration */ + #warning "Unexpected CONFIG_IDF_TARGET_NN value" #define NO_ESP32_CRYPT #define NO_WOLFSSL_ESP32_CRYPT_HASH #define NO_WOLFSSL_ESP32_CRYPT_AES @@ -392,12 +412,75 @@ #define ATCA_WOLFSSL */ -/* optional SM4 Ciphers. See https://github.com/wolfSSL/wolfsm +/***************************** Certificate Macros ***************************** + * + * The section below defines macros used in typically all of the wolfSSL + * examples such as the client and server for certs stored in header files. + * + * There are various certificate examples in this header file: + * https://github.com/wolfSSL/wolfssl/blob/master/wolfssl/certs_test.h + * + * To use the sets of macros below, define *one* of these: + * + * USE_CERT_BUFFERS_1024 - ECC 1024 bit encoded ASN1 + * USE_CERT_BUFFERS_2048 - RSA 2048 bit encoded ASN1 + * WOLFSSL_SM[2,3,4] - SM Ciphers + * + * For example: define USE_CERT_BUFFERS_2048 to use CA Certs used in this + * wolfSSL function for the `ca_cert_der_2048` buffer, size and types: + * + * ret = wolfSSL_CTX_load_verify_buffer(ctx, + * CTX_CA_CERT, + * CTX_CA_CERT_SIZE, + * CTX_CA_CERT_TYPE); + * + * See https://www.wolfssl.com/documentation/manuals/wolfssl/group__CertsKeys.html#function-wolfssl_ctx_load_verify_buffer + * + * In this case the CTX_CA_CERT will be defined as `ca_cert_der_2048` as + * defined here: https://github.com/wolfSSL/wolfssl/blob/master/wolfssl/certs_test.h + * + * The CTX_CA_CERT_SIZE and CTX_CA_CERT_TYPE are similarly used to reference + * array size and cert type respectively. + * + * Similarly for loading the private client key: + * + * ret = wolfSSL_CTX_use_PrivateKey_buffer(ctx, + * CTX_CLIENT_KEY, + * CTX_CLIENT_KEY_SIZE, + * CTX_CLIENT_KEY_TYPE); + * + * see https://www.wolfssl.com/documentation/manuals/wolfssl/group__CertsKeys.html#function-wolfssl_ctx_use_privatekey_buffer + * + * Similarly, the other macros are for server certificates and keys: + * `CTX_SERVER_CERT` and `CTX_SERVER_KEY` are available. + * + * The certificate and key names are typically `static const unsigned char` + * arrays. The [NAME]_size are typically `sizeof([array name])`, and the types + * are the known wolfSSL encoding type integers (e.g. WOLFSSL_FILETYPE_PEM). + * + * See `SSL_FILETYPE_[name]` in + * https://github.com/wolfSSL/wolfssl/blob/master/wolfssl/ssl.h + * + * See Abstract Syntax Notation One (ASN.1) in: + * https://github.com/wolfSSL/wolfssl/blob/master/wolfssl/wolfcrypt/asn.h + * + * Optional SM4 Ciphers: + * + * Although the SM ciphers are shown here, the `certs_test_sm.h` may not yet + * be available. See: + * https://github.com/wolfSSL/wolfssl/pull/6825 + * https://github.com/wolfSSL/wolfsm + * + * Uncomment these 3 macros to enable the SM Ciphers and use the macros below. + */ + +/* #define WOLFSSL_SM2 #define WOLFSSL_SM3 #define WOLFSSL_SM4 */ +/* Conditional macros used in wolfSSL TLS client and server examples */ #if defined(WOLFSSL_SM2) || defined(WOLFSSL_SM3) || defined(WOLFSSL_SM4) #include #define CTX_CA_CERT root_sm2 @@ -413,15 +496,47 @@ #undef WOLFSSL_BASE16 #define WOLFSSL_BASE16 #else - #define USE_CERT_BUFFERS_2048 - #define USE_CERT_BUFFERS_256 - #define CTX_CA_CERT ca_cert_der_2048 - #define CTX_CA_CERT_SIZE sizeof_ca_cert_der_2048 - #define CTX_CA_CERT_TYPE WOLFSSL_FILETYPE_ASN1 - #define CTX_SERVER_CERT server_cert_der_2048 - #define CTX_SERVER_CERT_SIZE sizeof_server_cert_der_2048 - #define CTX_SERVER_CERT_TYPE WOLFSSL_FILETYPE_ASN1 - #define CTX_SERVER_KEY server_key_der_2048 - #define CTX_SERVER_KEY_SIZE sizeof_server_key_der_2048 - #define CTX_SERVER_KEY_TYPE WOLFSSL_FILETYPE_ASN1 -#endif + #if defined(USE_CERT_BUFFERS_2048) + #include + #define CTX_CA_CERT ca_cert_der_2048 + #define CTX_CA_CERT_SIZE sizeof_ca_cert_der_2048 + #define CTX_CA_CERT_TYPE WOLFSSL_FILETYPE_ASN1 + + #define CTX_SERVER_CERT server_cert_der_2048 + #define CTX_SERVER_CERT_SIZE sizeof_server_cert_der_2048 + #define CTX_SERVER_CERT_TYPE WOLFSSL_FILETYPE_ASN1 + #define CTX_SERVER_KEY server_key_der_2048 + #define CTX_SERVER_KEY_SIZE sizeof_server_key_der_2048 + #define CTX_SERVER_KEY_TYPE WOLFSSL_FILETYPE_ASN1 + + #define CTX_CLIENT_CERT client_cert_der_2048 + #define CTX_CLIENT_CERT_SIZE sizeof_client_cert_der_2048 + #define CTX_CLIENT_CERT_TYPE WOLFSSL_FILETYPE_ASN1 + #define CTX_CLIENT_KEY client_key_der_2048 + #define CTX_CLIENT_KEY_SIZE sizeof_client_key_der_2048 + #define CTX_CLIENT_KEY_TYPE WOLFSSL_FILETYPE_ASN1 + + #elif defined(USE_CERT_BUFFERS_1024) + #include + #define CTX_CA_CERT ca_cert_der_1024 + #define CTX_CA_CERT_SIZE sizeof_ca_cert_der_1024 + #define CTX_CA_CERT_TYPE WOLFSSL_FILETYPE_ASN1 + + #define CTX_CLIENT_CERT client_cert_der_1024 + #define CTX_CLIENT_CERT_SIZE sizeof_client_cert_der_1024 + #define CTX_CLIENT_CERT_TYPE WOLFSSL_FILETYPE_ASN1 + #define CTX_CLIENT_KEY client_key_der_1024 + #define CTX_CLIENT_KEY_SIZE sizeof_client_key_der_1024 + #define CTX_CLIENT_KEY_TYPE WOLFSSL_FILETYPE_ASN1 + + #define CTX_SERVER_CERT server_cert_der_1024 + #define CTX_SERVER_CERT_SIZE sizeof_server_cert_der_1024 + #define CTX_SERVER_CERT_TYPE WOLFSSL_FILETYPE_ASN1 + #define CTX_SERVER_KEY server_key_der_1024 + #define CTX_SERVER_KEY_SIZE sizeof_server_key_der_1024 + #define CTX_SERVER_KEY_TYPE WOLFSSL_FILETYPE_ASN1 + #else + /* Optionally define custom cert arrays, sizes, and types here */ + #error "Must define USE_CERT_BUFFERS_2048 or USE_CERT_BUFFERS_1024" + #endif +#endif /* Conditional key and cert constant names */ diff --git a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/client-tls.c b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/client-tls.c index 9d5d26db..30388b31 100644 --- a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/client-tls.c +++ b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/client-tls.c @@ -1,6 +1,6 @@ /* client-tls.c * - * Copyright (C) 2006-2023 wolfSSL Inc. + * Copyright (C) 2006-2024 wolfSSL Inc. * * This file is part of wolfSSL. * @@ -18,7 +18,6 @@ * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA */ - #include "client-tls.h" /* Espressif FreeRTOS */ @@ -28,13 +27,15 @@ #include #endif +/* Espressif */ +#include + /* socket includes */ #include #include /* wolfSSL */ #include -#include "user_settings.h" #include #ifdef WOLFSSL_TRACK_MEMORY @@ -50,30 +51,6 @@ #define DEFAULT_MAX_DHKEY_BITS 2048 #endif -#if defined(WOLFSSL_SM2) || defined(WOLFSSL_SM3) || defined(WOLFSSL_SM4) - #include - #define CTX_CA_CERT root_sm2 - #define CTX_CA_CERT_SIZE sizeof_root_sm2 - #define CTX_CA_CERT_TYPE WOLFSSL_FILETYPE_PEM - #define CTX_CLIENT_CERT client_sm2 - #define CTX_CLIENT_CERT_SIZE sizeof_client_sm2 - #define CTX_CLIENT_CERT_TYPE WOLFSSL_FILETYPE_PEM - #define CTX_CLIENT_KEY client_sm2_priv - #define CTX_CLIENT_KEY_SIZE sizeof_client_sm2_priv - #define CTX_CLIENT_KEY_TYPE WOLFSSL_FILETYPE_PEM -#else - #include - #define CTX_CA_CERT ca_cert_der_2048 - #define CTX_CA_CERT_SIZE sizeof_ca_cert_der_2048 - #define CTX_CA_CERT_TYPE WOLFSSL_FILETYPE_ASN1 - #define CTX_CLIENT_CERT client_cert_der_2048 - #define CTX_CLIENT_CERT_SIZE sizeof_client_cert_der_2048 - #define CTX_CLIENT_CERT_TYPE WOLFSSL_FILETYPE_ASN1 - #define CTX_CLIENT_KEY client_key_der_2048 - #define CTX_CLIENT_KEY_SIZE sizeof_client_key_der_2048 - #define CTX_CLIENT_KEY_TYPE WOLFSSL_FILETYPE_ASN1 -#endif - /* Project */ #include "wifi_connect.h" #include "time_helper.h" @@ -87,7 +64,7 @@ * -h 192.168.1.128 -v 4 -l TLS13-SM4-CCM-SM3 -c ./certs/sm2/client-sm2.pem -k ./certs/sm2/client-sm2-priv.pem -A ./certs/sm2/root-sm2.pem -C * **/ -static const char* const TAG = "tls_client"; +#define TAG "client-tls" #if defined(DEBUG_WOLFSSL) int stack_start = -1; @@ -264,29 +241,29 @@ WOLFSSL_ESP_TASK tls_smp_client_task(void* args) * * reference code for SM Ciphers: * - #if defined(HAVE_AESGCM) && !defined(NO_DH) - #ifdef WOLFSSL_TLS13 - defaultCipherList = "TLS13-AES128-GCM-SHA256" - #ifndef WOLFSSL_NO_TLS12 - ":DHE-PSK-AES128-GCM-SHA256" - #endif - ; - #else - defaultCipherList = "DHE-PSK-AES128-GCM-SHA256"; + #if defined(HAVE_AESGCM) && !defined(NO_DH) + #ifdef WOLFSSL_TLS13 + defaultCipherList = "TLS13-AES128-GCM-SHA256" + #ifndef WOLFSSL_NO_TLS12 + ":DHE-PSK-AES128-GCM-SHA256" #endif - #elif defined(HAVE_AESGCM) && defined(WOLFSSL_TLS13) - defaultCipherList = "TLS13-AES128-GCM-SHA256:PSK-AES128-GCM-SHA256" - #ifndef WOLFSSL_NO_TLS12 - ":PSK-AES128-GCM-SHA256" - #endif - ; - #elif defined(HAVE_NULL_CIPHER) - defaultCipherList = "PSK-NULL-SHA256"; - #elif !defined(NO_AES_CBC) - defaultCipherList = "PSK-AES128-CBC-SHA256"; + ; #else - defaultCipherList = "PSK-AES128-GCM-SHA256"; + defaultCipherList = "DHE-PSK-AES128-GCM-SHA256"; #endif + #elif defined(HAVE_AESGCM) && defined(WOLFSSL_TLS13) + defaultCipherList = "TLS13-AES128-GCM-SHA256:PSK-AES128-GCM-SHA256" + #ifndef WOLFSSL_NO_TLS12 + ":PSK-AES128-GCM-SHA256" + #endif + ; + #elif defined(HAVE_NULL_CIPHER) + defaultCipherList = "PSK-NULL-SHA256"; + #elif !defined(NO_AES_CBC) + defaultCipherList = "PSK-AES128-CBC-SHA256"; + #else + defaultCipherList = "PSK-AES128-GCM-SHA256"; + #endif */ ret = wolfSSL_CTX_set_cipher_list(ctx, WOLFSSL_ESP32_CIPHER_SUITE); @@ -294,16 +271,16 @@ WOLFSSL_ESP_TASK tls_smp_client_task(void* args) ESP_LOGI(TAG, "Set cipher list: %s\n", WOLFSSL_ESP32_CIPHER_SUITE); } else { - ESP_LOGE(TAG, "ERROR: failed to set cipher list: %s\n", WOLFSSL_ESP32_CIPHER_SUITE); + ESP_LOGE(TAG, "ERROR: failed to set cipher list: %s\n", + WOLFSSL_ESP32_CIPHER_SUITE); } #endif #ifdef DEBUG_WOLFSSL ShowCiphers(NULL); - ESP_LOGI(TAG, - "Stack used: %d\n", - CONFIG_ESP_MAIN_TASK_STACK_SIZE - - uxTaskGetStackHighWaterMark(NULL)); + ESP_LOGI(TAG, "Stack used: %d\n", + CONFIG_ESP_MAIN_TASK_STACK_SIZE + - uxTaskGetStackHighWaterMark(NULL)); #endif /* see user_settings PROJECT_DH for HAVE_DH and HAVE_FFDHE_2048 */ @@ -328,12 +305,13 @@ WOLFSSL_ESP_TASK tls_smp_client_task(void* args) CTX_CLIENT_CERT_SIZE, CTX_CLIENT_CERT_TYPE); if (ret_i != SSL_SUCCESS) { - ESP_LOGE(TAG, "ERROR: failed to load chain %d, please check the file.\n", ret_i); + ESP_LOGE(TAG, "ERROR: failed to load chain %d, " + "please check the file.", ret_i); } - /* Load client certificates into WOLFSSL_CTX */ - WOLFSSL_MSG("Loading...cert"); - ret_i = wolfSSL_CTX_load_verify_buffer(ctx, + /* Load client certificates into WOLFSSL_CTX */ + WOLFSSL_MSG("Loading...cert"); + ret_i = wolfSSL_CTX_load_verify_buffer(ctx, CTX_CA_CERT, CTX_CA_CERT_SIZE, CTX_CA_CERT_TYPE); @@ -420,10 +398,17 @@ WOLFSSL_ESP_TASK tls_smp_client_task(void* args) #endif /* Attach wolfSSL to the socket */ - wolfSSL_set_fd(ssl, sockfd); + ret_i = wolfSSL_set_fd(ssl, sockfd); + if (ret_i == WOLFSSL_SUCCESS) { + ESP_LOGI(TAG, "wolfSSL_set_fd success"); + } + else { + ESP_LOGE(TAG, "ERROR: failed wolfSSL_set_fd. Error: %d\n", ret_i); + } WOLFSSL_MSG("Connect to wolfSSL on the server side"); /* Connect to wolfSSL on the server side */ + ret_i = wolfSSL_connect(ssl); if (wolfSSL_connect(ssl) == SSL_SUCCESS) { #ifdef DEBUG_WOLFSSL ShowCiphers(ssl); @@ -458,7 +443,8 @@ WOLFSSL_ESP_TASK tls_smp_client_task(void* args) printf("%s\n", buff); } else { - ESP_LOGE(TAG, "ERROR: failed to connect to wolfSSL\n"); + ESP_LOGE(TAG, "ERROR: failed to connect to wolfSSL. " + "Error: %d\n", ret_i); } #ifdef DEBUG_WOLFSSL ShowCiphers(ssl); @@ -487,16 +473,28 @@ WOLFSSL_ESP_TASK tls_smp_client_init(void* args) #else xTaskHandle _handle; #endif - /* http://esp32.info/docs/esp_idf/html/dd/d3c/group__xTaskCreate.html */ + /* See https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/system/freertos_idf.html#functions */ + if (TLS_SMP_CLIENT_TASK_BYTES < (6 * 1024)) { + /* Observed approximately 6KB limit for the RTOS task stack size. + * Reminder parameter is bytes, not words as with generic FreeeRTOS. */ + ESP_LOGW(TAG, "Warning: TLS_SMP_CLIENT_TASK_BYTES < 6KB"); + } +#ifndef WOLFSSL_SMALL_STACK + ESP_LOGW(TAG, "WARNING: WOLFSSL_SMALL_STACK is not defined. Consider " + "defining that to reduce embedded memory usage."); +#endif + + /* Note that despite vanilla FreeRTOS using WORDS for a parameter, + * Espressif uses BYTES for the task stack size here: */ ret = xTaskCreate(tls_smp_client_task, TLS_SMP_CLIENT_TASK_NAME, - TLS_SMP_CLIENT_TASK_WORDS, + TLS_SMP_CLIENT_TASK_BYTES, NULL, TLS_SMP_CLIENT_TASK_PRIORITY, &_handle); if (ret != pdPASS) { - ESP_LOGI(TAG, "create thread %s failed", TLS_SMP_CLIENT_TASK_NAME); + ESP_LOGI(TAG, "Create thread %s failed.", TLS_SMP_CLIENT_TASK_NAME); } return TLS_SMP_CLIENT_TASK_RET; } diff --git a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/component.mk b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/component.mk index 61f8990c..c59edbee 100644 --- a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/component.mk +++ b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/component.mk @@ -1,8 +1,8 @@ # # Main component makefile. # -# This Makefile can be left empty. By default, it will take the sources in the -# src/ directory, compile them and link them into lib(subdirectory_name).a -# in the build directory. This behaviour is entirely configurable, +# This Makefile can be left empty. By default, it will take the sources in the +# src/ directory, compile them and link them into lib(subdirectory_name).a +# in the build directory. This behavior is entirely configurable, # please read the ESP-IDF documents if you need to do this. # diff --git a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/include/client-tls.h b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/include/client-tls.h index 1188ee36..de534035 100644 --- a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/include/client-tls.h +++ b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/include/client-tls.h @@ -1,6 +1,6 @@ -/* server-tls.h +/* client-tls.h * - * Copyright (C) 2006-2023 wolfSSL Inc. + * Copyright (C) 2006-2024 wolfSSL Inc. * * This file is part of wolfSSL. * @@ -18,18 +18,20 @@ * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA */ -#ifndef _SERVER_TLS_ -#define _SERVER_TLS_ +#ifndef _CLIENT_TLS_H_ +#define _CLIENT_TLS_H_ + +/* Local project, auto-generated configuration */ +#include "sdkconfig.h" #include #include -#include "sdkconfig.h" /* See main/Kconfig.projbuild for default configuration settings */ #ifdef CONFIG_WOLFSSL_TARGET_HOST #define TLS_SMP_TARGET_HOST CONFIG_WOLFSSL_TARGET_HOST #else - #define TLS_SMP_TARGET_HOST "192.168.1.38" + #define TLS_SMP_TARGET_HOST "192.168.1.37" #endif #ifdef CONFIG_WOLFSSL_TARGET_PORT @@ -39,13 +41,20 @@ #endif #define TLS_SMP_CLIENT_TASK_NAME "tls_client_example" -#define TLS_SMP_CLIENT_TASK_WORDS 22240 + +/* Reminder: Vanilla FreeRTOS is words, Espressif is bytes. */ +#if defined(WOLFSSL_ESP8266) + #define TLS_SMP_CLIENT_TASK_BYTES (6 * 1024) +#else + #define TLS_SMP_CLIENT_TASK_BYTES (8 * 1024) +#endif + #define TLS_SMP_CLIENT_TASK_PRIORITY 8 #if defined(SINGLE_THREADED) #define WOLFSSL_ESP_TASK int #else - #include "freertos/FreeRTOS.h" + #include #define WOLFSSL_ESP_TASK void #endif @@ -68,4 +77,5 @@ WOLFSSL_ESP_TASK tls_smp_client_task(void* args); #else WOLFSSL_ESP_TASK tls_smp_client_init(void* args); #endif + #endif /* _SERVER_TLS_ */ diff --git a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/include/main.h b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/include/main.h index 94c3b5eb..12c452d6 100644 --- a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/include/main.h +++ b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/include/main.h @@ -1,6 +1,6 @@ -/* template main.h +/* wolfssl_client main.h * - * Copyright (C) 2006-2023 wolfSSL Inc. + * Copyright (C) 2006-2024 wolfSSL Inc. * * This file is part of wolfSSL. * diff --git a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/include/time_helper.h b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/include/time_helper.h index a47f9400..3586ac65 100644 --- a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/include/time_helper.h +++ b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/include/time_helper.h @@ -1,5 +1,6 @@ -/* - * Copyright (C) 2006-2023 wolfSSL Inc. +/* time_helper.h + * + * Copyright (C) 2006-2024 wolfSSL Inc. * * This file is part of wolfSSL. * @@ -20,8 +21,8 @@ /* common Espressif time_helper v5.6.3.001 */ -#ifndef _TIME_HELPER_H -#define _TIME_HELPER_H +#ifndef _TIME_HELPER_H_ +#define _TIME_HELPER_H_ /* ESP-IDF uses a 64-bit signed integer to represent time_t starting from release v5.0 * See: https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/system/system_time.html#year-2036-and-2038-overflow-issues @@ -32,13 +33,13 @@ extern "C" { #endif /* a function to show the current data and time */ -int esp_show_current_datetime(); +int esp_show_current_datetime(void); /* worst case, if GitHub time not available, used fixed time */ int set_fixed_default_time(void); /* set time from string (e.g. GitHub commit time) */ -int set_time_from_string(char* time_buffer); +int set_time_from_string(const char* time_buffer); /* set time from NTP servers, * also initially calls set_fixed_default_time or set_time_from_string */ @@ -51,4 +52,4 @@ int set_time_wait_for_ntp(void); } /* extern "C" */ #endif -#endif /* #ifndef _TIME_HELPER_H */ +#endif /* #ifndef _TIME_HELPER_H_ */ diff --git a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/include/wifi_connect.h b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/include/wifi_connect.h index 644ce00d..b29d5812 100644 --- a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/include/wifi_connect.h +++ b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/include/wifi_connect.h @@ -21,9 +21,6 @@ #ifndef _WIFI_CONNECT_H_ #define _WIFI_CONNECT_H_ -#include -#include - /* ESP lwip */ #define EXAMPLE_ESP_MAXIMUM_RETRY CONFIG_ESP_MAXIMUM_RETRY @@ -48,19 +45,54 @@ * file my_private_config.h should be excluded from git updates */ /* #define USE_MY_PRIVATE_CONFIG */ -#ifdef USE_MY_PRIVATE_CONFIG +/* Note that IntelliSense may not work properly in the next section for the + * Espressif SDK 3.4 on the ESP8266. Macros should still be defined. + * See the project-level Makefile. Example found in: + * https://github.com/wolfSSL/wolfssl/tree/master/IDE/Espressif/ESP-IDF/examples/template + * + * The USE_MY_PRIVATE_[OS]_CONFIG is typically an environment variable that + * triggers the make (not cmake) to add compiler defines. + */ +#if defined(USE_MY_PRIVATE_WINDOWS_CONFIG) + #include "/workspace/my_private_config.h" +#elif defined(USE_MY_PRIVATE_WSL_CONFIG) + #include "/mnt/c/workspace/my_private_config.h" +#elif defined(USE_MY_PRIVATE_LINUX_CONFIG) + #include "~/workspace/my_private_config.h" +#elif defined(USE_MY_PRIVATE_MAC_CONFIG) + #include "~/Documents/my_private_config.h" +#elif defined(USE_MY_PRIVATE_CONFIG) + /* This section works best with cmake & non-environment variable setting */ #if defined(WOLFSSL_CMAKE_SYSTEM_NAME_WINDOWS) + #define WOLFSSL_CMAKE + #include "/workspace/my_private_config.h" + #elif defined(WOLFSSL_MAKE_SYSTEM_NAME_WINDOWS) + #define WOLFSSL_MAKE #include "/workspace/my_private_config.h" #elif defined(WOLFSSL_CMAKE_SYSTEM_NAME_WSL) + #define WOLFSSL_CMAKE + #include "/mnt/c/workspace/my_private_config.h" + #elif defined(WOLFSSL_MAKE_SYSTEM_NAME_WSL) + #define WOLFSSL_MAKE #include "/mnt/c/workspace/my_private_config.h" #elif defined(WOLFSSL_CMAKE_SYSTEM_NAME_LINUX) + #define WOLFSSL_CMAKE + #include "~/workspace/my_private_config.h" + #elif defined(WOLFSSL_MAKE_SYSTEM_NAME_LINUX) + #define WOLFSSL_MAKE #include "~/workspace/my_private_config.h" #elif defined(WOLFSSL_CMAKE_SYSTEM_NAME_APPLE) #include "~/Documents/my_private_config.h" + #elif defined(WOLFSSL_MAKE_SYSTEM_NAME_APPLE) + #define WOLFSSL_MAKE + #include "~/Documents/my_private_config.h" + #elif defined(OS_WINDOWS) + #include "/workspace/my_private_config.h" #else - #warning "did not detect environment. using ~/my_private_config.h" - #include "~/my_private_config.h" - #endif + /* Edit as needed for your private config: */ + #warning "default private config using /workspace/my_private_config.h" + #include "/workspace/my_private_config.h" + #endif #else /* @@ -70,14 +102,22 @@ ** If you'd rather not, just change the below entries to strings with ** the config you want - ie #define EXAMPLE_WIFI_SSID "mywifissid" */ - #ifdef CONFIG_ESP_WIFI_SSID + #if defined(CONFIG_ESP_WIFI_SSID) + /* tyically from ESP32 with ESP-IDF v4 ot v5 */ #define EXAMPLE_ESP_WIFI_SSID CONFIG_ESP_WIFI_SSID + #elif defined(CONFIG_EXAMPLE_WIFI_SSID) + /* tyically from ESP8266 rtos-sdk/v3.4 */ + #define EXAMPLE_ESP_WIFI_SSID CONFIG_EXAMPLE_WIFI_SSID #else #define EXAMPLE_ESP_WIFI_SSID "MYSSID_WIFI_CONNECT" #endif - #ifdef CONFIG_ESP_WIFI_PASSWORD + #if defined(CONFIG_ESP_WIFI_PASSWORD) + /* tyically from ESP32 with ESP-IDF v4 or v5 */ #define EXAMPLE_ESP_WIFI_PASS CONFIG_ESP_WIFI_PASSWORD + #elif defined(CONFIG_EXAMPLE_WIFI_SSID) + /* tyically from ESP8266 rtos-sdk/v3.4 */ + #define EXAMPLE_ESP_WIFI_PASS CONFIG_EXAMPLE_WIFI_PASSWORD #else #define EXAMPLE_ESP_WIFI_PASS "MYPASSWORD_WIFI_CONNECT" #endif diff --git a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/main.c b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/main.c index add43ada..fa116ed1 100644 --- a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/main.c +++ b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/main.c @@ -1,6 +1,6 @@ /* main.c * - * Copyright (C) 2006-2023 wolfSSL Inc. + * Copyright (C) 2006-2024 wolfSSL Inc. * * This file is part of wolfSSL. * @@ -27,8 +27,9 @@ #include /* wolfSSL */ +/* Always include wolfcrypt/settings.h before any other wolfSSL file. */ +/* Reminder: settings.h pulls in user_settings.h; don't include it here */ #include -#include #include #ifndef WOLFSSL_ESPIDF #warning "Problem with wolfSSL user_settings." @@ -44,13 +45,17 @@ * For wired ethernet, see: * https://github.com/wolfSSL/wolfssl-examples/tree/master/ESP32/TLS13-ENC28J60-client */ #include "wifi_connect.h" + /* + * Note ModBus TCP cannot be disabled on ESP8266 tos-sdk/v3.4 + * See https://github.com/espressif/esp-modbus/issues/2 + */ #endif #ifdef WOLFSSL_TRACK_MEMORY #include #endif -static const char* const TAG = "TLS Client"; +static const char* TAG = "main"; #if defined(WOLFSSL_ESPWROOM32SE) && defined(HAVE_PK_CALLBACKS) \ && defined(WOLFSSL_ATECC508A) @@ -115,7 +120,7 @@ void my_atmel_free(int slotId) #endif /* CUSTOM_SLOT_ALLOCATION */ #endif /* WOLFSSL_ESPWROOM32SE && HAVE_PK_CALLBACK && WOLFSSL_ATECC508A */ -/* for FreeRTOS */ +/* Entry for FreeRTOS */ void app_main(void) { int stack_start = 0; @@ -126,26 +131,42 @@ void app_main(void) ESP_LOGI(TAG, "---------------------- BEGIN MAIN ----------------------"); ESP_LOGI(TAG, "--------------------------------------------------------"); ESP_LOGI(TAG, "--------------------------------------------------------"); +#ifdef ESP_SDK_MEM_LIB_VERSION + sdk_init_meminfo(); +#endif #ifdef ESP_TASK_MAIN_STACK ESP_LOGI(TAG, "ESP_TASK_MAIN_STACK: %d", ESP_TASK_MAIN_STACK); #endif #ifdef TASK_EXTRA_STACK_SIZE ESP_LOGI(TAG, "TASK_EXTRA_STACK_SIZE: %d", TASK_EXTRA_STACK_SIZE); #endif -#ifdef INCLUDE_uxTaskGetStackHighWaterMark + +#ifdef SINGLE_THREADED + ESP_LOGI(TAG, "Single threaded"); +#else ESP_LOGI(TAG, "CONFIG_ESP_MAIN_TASK_STACK_SIZE = %d bytes (%d words)", CONFIG_ESP_MAIN_TASK_STACK_SIZE, - (int)(CONFIG_ESP_MAIN_TASK_STACK_SIZE / sizeof(void*))); + (int)(CONFIG_ESP_MAIN_TASK_STACK_SIZE / sizeof(void*))); + + #ifdef INCLUDE_uxTaskGetStackHighWaterMark + { + /* Returns the high water mark of the stack associated with xTask. That is, + * the minimum free stack space there has been (in bytes not words, unlike + * vanilla FreeRTOS) since the task started. The smaller the returned + * number the closer the task has come to overflowing its stack. + * see https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/system/freertos_idf.html + */ + stack_start = uxTaskGetStackHighWaterMark(NULL); + #ifdef ESP_SDK_MEM_LIB_VERSION + { + sdk_var_whereis("stack_start", &stack_start); + } + #endif - /* Returns the high water mark of the stack associated with xTask. That is, - * the minimum free stack space there has been (in bytes not words, unlike - * vanilla FreeRTOS) since the task started. The smaller the returned - * number the closer the task has come to overflowing its stack. - * see https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/system/freertos_idf.html - */ - stack_start = uxTaskGetStackHighWaterMark(NULL); - ESP_LOGI(TAG, "Stack Start HWM: %d bytes", stack_start); -#endif + ESP_LOGI(TAG, "Stack Start HWM: %d bytes", stack_start); + } + #endif /* INCLUDE_uxTaskGetStackHighWaterMark */ +#endif /* SINGLE_THREADED */ #ifdef HAVE_VERSION_EXTENDED_INFO esp_ShowExtendedSystemInfo(); @@ -184,11 +205,23 @@ void app_main(void) /* Initialize NVS */ ret = nvs_flash_init(); - if (ret == ESP_ERR_NVS_NO_FREE_PAGES || - ret == ESP_ERR_NVS_NEW_VERSION_FOUND) { - ESP_ERROR_CHECK(nvs_flash_erase()); - ret = nvs_flash_init(); + #if defined(CONFIG_IDF_TARGET_ESP8266) + { + if (ret == ESP_ERR_NVS_NO_FREE_PAGES) { + ESP_ERROR_CHECK(nvs_flash_erase()); + ret = nvs_flash_init(); + } } + #else + { + /* Non-ESP8266 initialization is slightly different */ + if (ret == ESP_ERR_NVS_NO_FREE_PAGES || + ret == ESP_ERR_NVS_NEW_VERSION_FOUND) { + ESP_ERROR_CHECK(nvs_flash_erase()); + ret = nvs_flash_init(); + } + } + #endif /* else not CONFIG_IDF_TARGET_ESP8266 */ ESP_ERROR_CHECK(ret); #if defined(CONFIG_IDF_TARGET_ESP32H2) @@ -203,8 +236,8 @@ void app_main(void) ESP_LOGI(TAG, "Trying WiFi again..."); ret = wifi_init_sta(); } - #endif -#endif + #endif /* else not CONFIG_IDF_TARGET_ESP32H2 */ +#endif /* else FOUND_PROTOCOL_EXAMPLES_DIR not found */ /* Once we are connected to the network, start & wait for NTP time */ ret = set_time_wait_for_ntp(); @@ -216,14 +249,6 @@ void app_main(void) esp_show_current_datetime(); } - /* HWM is maximum amount of stack space that has been unused, in bytes - * not words (unlike vanilla freeRTOS). */ - ESP_LOGI(TAG, "Initial Stack Used (before wolfSSL Server): %d bytes", - CONFIG_ESP_MAIN_TASK_STACK_SIZE - - (uxTaskGetStackHighWaterMark(NULL)) - ); - ESP_LOGI(TAG, "Starting TLS Client task ...\n"); - #if defined(SINGLE_THREADED) /* just call the task */ tls_smp_client_task((void*)NULL); @@ -232,6 +257,19 @@ void app_main(void) /* start a thread with the task */ args[0].loops = 10; args[0].port = 11111; + + /* HWM is maximum amount of stack space that has been unused, in bytes + * not words (unlike vanilla freeRTOS). */ + int this_heap; + this_heap = esp_get_free_heap_size(); + ESP_LOGI(TAG, "Initial Stack Used (before wolfSSL Server): %d bytes", + CONFIG_ESP_MAIN_TASK_STACK_SIZE + - (uxTaskGetStackHighWaterMark(NULL)) + ); + ESP_LOGI(TAG, "Starting TLS Client task ...\n"); + + ESP_LOGI(TAG, "main tls_smp_client_init heap @ %p = %d", + &this_heap, this_heap); tls_smp_client_init(args); /* optional additional client threads tls_smp_client_init(args); @@ -244,24 +282,24 @@ void app_main(void) */ #endif + /* Done */ +#ifdef SINGLE_THREADED + ESP_LOGV(TAG, "\n\nDone!\n\n"); + while (1); +#else ESP_LOGV(TAG, "\n\nvTaskDelete...\n\n"); vTaskDelete(NULL); /* done */ while (1) { ESP_LOGV(TAG, "\n\nLoop...\n\n"); -#ifdef INCLUDE_uxTaskGetStackHighWaterMark + #ifdef INCLUDE_uxTaskGetStackHighWaterMark ESP_LOGI(TAG, "Stack HWM: %d", uxTaskGetStackHighWaterMark(NULL)); ESP_LOGI(TAG, "Stack used: %d", CONFIG_ESP_MAIN_TASK_STACK_SIZE - (uxTaskGetStackHighWaterMark(NULL) )); -#endif - -#if defined(SINGLE_THREADED) - ESP_LOGV(TAG, "\n\nDone!\n\n"); - while (1); -#else + #endif vTaskDelay(60000); -#endif - } /* done whle */ + } /* done while */ +#endif /* else not SINGLE_THREADED */ } /* app_main */ diff --git a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/time_helper.c b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/time_helper.c index 5149d2e6..5eb06a14 100644 --- a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/time_helper.c +++ b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/time_helper.c @@ -1,6 +1,6 @@ /* time_helper.c * - * Copyright (C) 2006-2023 wolfSSL Inc. + * Copyright (C) 2006-2024 wolfSSL Inc. * * This file is part of wolfSSL. * @@ -19,12 +19,14 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA */ -/* common Espressif time_helper v5.6.3.002 */ -#include "esp_idf_version.h" +/* See https://tf.nist.gov/tf-cgi/servers.cgi */ + +/* common Espressif time_helper v5.6.6.001 */ #include "sdkconfig.h" #include "time_helper.h" #include +#include #if defined(ESP_IDF_VERSION_MAJOR) && defined(ESP_IDF_VERSION_MINOR) #if (ESP_IDF_VERSION_MAJOR == 5) && (ESP_IDF_VERSION_MINOR >= 1) @@ -36,25 +38,24 @@ #include #endif #else - /* TODO Consider pre IDF v5? */ + /* TODO Consider non ESP-IDF environments */ #endif /* ESP-IDF uses a 64-bit signed integer to represent time_t starting from release v5.0 * See: https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/system/system_time.html#year-2036-and-2038-overflow-issues */ -const static char* TAG = "time_helper"; /* see https://www.gnu.org/software/libc/manual/html_node/TZ-Variable.html */ #ifndef TIME_ZONE -/* - * PST represents Pacific Standard Time. - * +8 specifies the offset from UTC (Coordinated Universal Time), indicating - * that Pacific Time is UTC-8 during standard time. - * PDT represents Pacific Daylight Time. - * M3.2.0 indicates that Daylight Saving Time (DST) starts on the - * second (2) Sunday (0) of March (3). - * M11.1.0 indicates that DST ends on the first (1) Sunday (0) of November (11) - */ + /* + * PST represents Pacific Standard Time. + * +8 specifies the offset from UTC (Coordinated Universal Time), indicating + * that Pacific Time is UTC-8 during standard time. + * PDT represents Pacific Daylight Time. + * M3.2.0 indicates that Daylight Saving Time (DST) starts on the + * second (2) Sunday (0) of March (3). + * M11.1.0 indicates that DST ends on the first (1) Sunday (0) of November (11) + */ #define TIME_ZONE "PST+8PDT,M3.2.0,M11.1.0" #endif /* not defined: TIME_ZONE, so we are setting our own */ @@ -87,11 +88,13 @@ const static char* TAG = "time_helper"; char* ntpServerList[NTP_SERVER_COUNT] = NTP_SERVER_LIST; +const static char* TAG = "time_helper"; + /* our NTP server list is global info */ extern char* ntpServerList[NTP_SERVER_COUNT]; /* Show the current date and time */ -int esp_show_current_datetime() +int esp_show_current_datetime(void) { time_t now; char strftime_buf[64]; @@ -104,7 +107,7 @@ int esp_show_current_datetime() localtime_r(&now, &timeinfo); strftime(strftime_buf, sizeof(strftime_buf), "%c", &timeinfo); ESP_LOGI(TAG, "The current date/time is: %s", strftime_buf); - return 0; + return ESP_OK; } /* the worst-case scenario is a hard-coded date/time */ @@ -113,9 +116,9 @@ int set_fixed_default_time(void) /* ideally, we'd like to set time from network, * but let's set a default time, just in case */ struct tm timeinfo = { - .tm_year = 2023 - 1900, - .tm_mon = 10, - .tm_mday = 02, + .tm_year = 2024 - 1900, + .tm_mon = 1, + .tm_mday = 05, .tm_hour = 13, .tm_min = 01, .tm_sec = 05 @@ -130,7 +133,38 @@ int set_fixed_default_time(void) ESP_LOGI(TAG, "Adjusting time from fixed value"); now = (struct timeval){ .tv_sec = interim_time }; ret = settimeofday(&now, NULL); + ESP_LOGI(TAG, "settimeofday result = %d", ret); + return ret; +} + +/* probably_valid_time_string(s) + * + * some sanity checks on time string before calling sscanf() + * + * returns 0 == ESP_OK == Success if str is likely a valid time. + * -1 == ESP_FAIL otherwise + */ +int probably_valid_time_string(const char* str) +{ + int ret = ESP_OK; + size_t length = 0; + size_t spaces = 0; + size_t colons = 0; + + while (str[length] != '\0') { + if (str[length] == ' ') { + spaces++; + } + if (str[length] == ':') { + colons++; + } + length++; + } + if ((length > 32) || (spaces < 4) || (spaces > 5) || (colons > 2)) { + ret = ESP_FAIL; + ESP_LOGE(TAG, "ERROR, failed time sanity check: %s", str); + } return ret; } @@ -138,60 +172,66 @@ int set_fixed_default_time(void) * * returns 0 = success if able to set the time from the provided string * error for any other value, typically -1 */ -int set_time_from_string(char* time_buffer) +int set_time_from_string(const char* time_buffer) { /* expecting github default formatting: 'Thu Aug 31 12:41:45 2023 -0700' */ + char offset[28]; /* large arrays, just in case there's still bad data */ + char day_str[28]; + char month_str[28]; const char *format = "%3s %3s %d %d:%d:%d %d %s"; struct tm this_timeinfo; struct timeval now; time_t interim_time; - char offset[6]; /* expecting trailing single quote, not used */ - char day_str[4]; - char month_str[4]; int day, year, hour, minute, second; int quote_offset = 0; int ret = 0; - /* we are expecting the string to be encapsulated in single quotes */ - if (*time_buffer == 0x27) { - quote_offset = 1; - } + /* perform some basic sanity checkes */ + ret = probably_valid_time_string(time_buffer); + if (ret == ESP_OK) { + /* we are expecting the string to be encapsulated in single quotes */ + if (*time_buffer == 0x27) { + quote_offset = 1; + } - ret = sscanf(time_buffer + quote_offset, - format, - day_str, month_str, - &day, &hour, &minute, &second, &year, &offset); + ret = sscanf(time_buffer + quote_offset, + format, + day_str, month_str, + &day, &hour, &minute, &second, &year, &offset); - if (ret == 8) { - /* we found a match for all componets */ + if (ret == 8) { + /* we found a match for all componets */ - const char *months[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", - "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; + const char *months[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; - for (int i = 0; i < 12; i++) { - if (strcmp(month_str, months[i]) == 0) { - this_timeinfo.tm_mon = i; - break; + for (int i = 0; i < 12; i++) { + if (strcmp(month_str, months[i]) == 0) { + this_timeinfo.tm_mon = i; + break; + } } - } - this_timeinfo.tm_mday = day; - this_timeinfo.tm_hour = hour; - this_timeinfo.tm_min = minute; - this_timeinfo.tm_sec = second; - this_timeinfo.tm_year = year - 1900; /* Number of years since 1900 */ + this_timeinfo.tm_mday = day; + this_timeinfo.tm_hour = hour; + this_timeinfo.tm_min = minute; + this_timeinfo.tm_sec = second; + this_timeinfo.tm_year = year - 1900; /* Number of years since 1900 */ - interim_time = mktime(&this_timeinfo); - now = (struct timeval){ .tv_sec = interim_time }; - ret = settimeofday(&now, NULL); - ESP_LOGI(TAG, "Time updated to %s", time_buffer); - } - else { - ESP_LOGE(TAG, "Failed to convert \"%s\" to a tm date.", time_buffer); - ESP_LOGI(TAG, "Trying fixed date that was hard-coded."); - set_fixed_default_time(); - ret = -1; + interim_time = mktime(&this_timeinfo); + now = (struct timeval){ .tv_sec = interim_time }; + ret = settimeofday(&now, NULL); + ESP_LOGI(TAG, "Time updated to %s", time_buffer); + } + else { + ESP_LOGE(TAG, "Failed to convert \"%s\" to a tm date.", + time_buffer); + ESP_LOGI(TAG, "Trying fixed date that was hard-coded...."); + set_fixed_default_time(); + ret = ESP_FAIL; + } } + return ret; } @@ -224,14 +264,16 @@ int set_time(void) #ifdef LIBWOLFSSL_VERSION_GIT_HASH_DATE /* initialy set a default approximate time from recent git commit */ - ESP_LOGI(TAG, "Found git hash date, attempting to set system date."); - set_time_from_string(LIBWOLFSSL_VERSION_GIT_HASH_DATE); + ESP_LOGI(TAG, "Found git hash date, attempting to set system date: %s", + LIBWOLFSSL_VERSION_GIT_HASH_DATE); + set_time_from_string(LIBWOLFSSL_VERSION_GIT_HASH_DATE"\0"); esp_show_current_datetime(); ret = -4; #else /* otherwise set a fixed time that was hard coded */ set_fixed_default_time(); + esp_show_current_datetime(); ret = -3; #endif @@ -262,6 +304,7 @@ int set_time(void) } ESP_LOGI(TAG, "%s", thisServer); sntp_setservername(i, thisServer); + ret = ESP_OK; } #ifdef HAS_ESP_NETIF_SNTP ret = esp_netif_sntp_init(&config); @@ -289,6 +332,9 @@ int set_time(void) ESP_LOGW(TAG, "No sntp time servers found."); ret = -1; } + + esp_show_current_datetime(); + ESP_LOGI(TAG, "time helper existing with result = %d", ret); return ret; } @@ -303,6 +349,8 @@ int set_time_wait_for_ntp(void) ret = esp_netif_sntp_start(); ret = esp_netif_sntp_sync_wait(500 / portTICK_PERIOD_MS); +#else + ESP_LOGE(TAG, "HAS_ESP_NETIF_SNTP not defined"); #endif /* HAS_ESP_NETIF_SNTP */ esp_show_current_datetime(); diff --git a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/wifi_connect.c b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/wifi_connect.c index b9f9ab73..19ced330 100644 --- a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/wifi_connect.c +++ b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/wifi_connect.c @@ -1,6 +1,6 @@ /* wifi_connect.c * - * Copyright (C) 2006-2023 wolfSSL Inc. + * Copyright (C) 2006-2024 wolfSSL Inc. * * This file is part of wolfSSL. * @@ -18,17 +18,20 @@ * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA */ - #include "wifi_connect.h" +#include "wifi_connect.h" +/* FreeRTOS */ #include #include #include -#include + +/* Espressif */ #include +#include +#include /* wolfSSL */ #include -#include "user_settings.h" #include #include #ifndef WOLFSSL_ESPIDF @@ -36,7 +39,12 @@ #warning "Check components/wolfssl/include" #endif -#if ESP_IDF_VERSION_MAJOR >= 5 +/* When there's too little heap, WiFi quietly refuses to connect */ +#define WIFI_LOW_HEAP_WARNING 21132 + +#if defined(CONFIG_IDF_TARGET_ESP8266) +#elif ESP_IDF_VERSION_MAJOR >= 5 + /* example path set in cmake file */ #elif ESP_IDF_VERSION_MAJOR >= 4 #include "protocol_examples_common.h" #else @@ -44,7 +52,9 @@ static EventGroupHandle_t wifi_event_group; #endif -#if defined(ESP_IDF_VERSION_MAJOR) && defined(ESP_IDF_VERSION_MINOR) +#if defined(CONFIG_IDF_TARGET_ESP8266) + +#elif defined(ESP_IDF_VERSION_MAJOR) && defined(ESP_IDF_VERSION_MINOR) #if ESP_IDF_VERSION_MAJOR >= 4 /* likely using examples, see wifi_connect.h */ #else @@ -64,7 +74,114 @@ /* breadcrumb prefix for logging */ const static char *TAG = "wifi_connect"; -#if ESP_IDF_VERSION_MAJOR < 4 +#if defined(CONFIG_IDF_TARGET_ESP8266) +#ifndef CONFIG_ESP_MAX_STA_CONN + #define CONFIG_ESP_MAX_STA_CONN 4 +#endif +#define EXAMPLE_MAX_STA_CONN CONFIG_ESP_MAX_STA_CONN + +#define WIFI_CONNECTED_BIT BIT0 +#define WIFI_FAIL_BIT BIT1 +#ifndef CONFIG_ESP_MAXIMUM_RETRY + #define CONFIG_ESP_MAXIMUM_RETRY 5 +#endif +/* FreeRTOS event group to signal when we are connected*/ +static EventGroupHandle_t s_wifi_event_group; +static int s_retry_num = 0; + +#define EXAMPLE_ESP_MAXIMUM_RETRY CONFIG_ESP_MAXIMUM_RETRY +static void event_handler(void* arg, esp_event_base_t event_base, + int32_t event_id, void* event_data) +{ + if (event_base == WIFI_EVENT && event_id == WIFI_EVENT_STA_START) { + esp_wifi_connect(); + } else if (event_base == WIFI_EVENT && event_id == WIFI_EVENT_STA_DISCONNECTED) { + if (s_retry_num < EXAMPLE_ESP_MAXIMUM_RETRY) { + esp_wifi_connect(); + s_retry_num++; + ESP_LOGI(TAG, "retry to connect to the AP"); + } else { + xEventGroupSetBits(s_wifi_event_group, WIFI_FAIL_BIT); + } + ESP_LOGI(TAG,"connect to the AP fail"); + } else if (event_base == IP_EVENT && event_id == IP_EVENT_STA_GOT_IP) { + ip_event_got_ip_t* event = (ip_event_got_ip_t*) event_data; + ESP_LOGI(TAG, "got ip:%s", + ip4addr_ntoa(&event->ip_info.ip)); + s_retry_num = 0; + xEventGroupSetBits(s_wifi_event_group, WIFI_CONNECTED_BIT); + } +} + +int wifi_init_sta(void) +{ + word32 this_heap; + + s_wifi_event_group = xEventGroupCreate(); + + tcpip_adapter_init(); + + ESP_ERROR_CHECK(esp_event_loop_create_default()); + + wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT(); + ESP_ERROR_CHECK(esp_wifi_init(&cfg)); + + ESP_ERROR_CHECK(esp_event_handler_register(WIFI_EVENT, ESP_EVENT_ANY_ID, &event_handler, NULL)); + ESP_ERROR_CHECK(esp_event_handler_register(IP_EVENT, IP_EVENT_STA_GOT_IP, &event_handler, NULL)); + + wifi_config_t wifi_config = { + .sta = { + .ssid = EXAMPLE_ESP_WIFI_SSID, + .password = EXAMPLE_ESP_WIFI_PASS + }, + }; + + /* Setting a password implies station will connect to all security modes including WEP/WPA. + * However these modes are deprecated and not advisable to be used. Incase your Access point + * doesn't support WPA2, these mode can be enabled by commenting below line */ + + if (strlen((char *)wifi_config.sta.password)) { + wifi_config.sta.threshold.authmode = WIFI_AUTH_WPA2_PSK; + } + + ESP_ERROR_CHECK(esp_wifi_set_mode(WIFI_MODE_STA) ); + ESP_ERROR_CHECK(esp_wifi_set_config(ESP_IF_WIFI_STA, &wifi_config) ); + ESP_ERROR_CHECK(esp_wifi_start() ); + + ESP_LOGI(TAG, "wifi_init_sta finished. Connecting..."); + this_heap = esp_get_free_heap_size(); + ESP_LOGI(TAG, "this heap = %d", this_heap); + if (this_heap < WIFI_LOW_HEAP_WARNING) { + ESP_LOGW(TAG, "Warning: WiFi low heap: %d", WIFI_LOW_HEAP_WARNING); + } + /* Waiting until either the connection is established (WIFI_CONNECTED_BIT) or connection failed for the maximum + * number of re-tries (WIFI_FAIL_BIT). The bits are set by event_handler() (see above) */ + EventBits_t bits = xEventGroupWaitBits(s_wifi_event_group, + WIFI_CONNECTED_BIT | WIFI_FAIL_BIT, + pdFALSE, + pdFALSE, + portMAX_DELAY); + + ESP_LOGI(TAG, "xEventGroupWaitBits finished."); + /* xEventGroupWaitBits() returns the bits before the call returned, hence we can test which event actually + * happened. */ + if (bits & WIFI_CONNECTED_BIT) { + ESP_LOGI(TAG, "connected to ap SSID:%s", + EXAMPLE_ESP_WIFI_SSID); + } else if (bits & WIFI_FAIL_BIT) { + ESP_LOGI(TAG, "Failed to connect to SSID:%s, password:%s", + EXAMPLE_ESP_WIFI_SSID, EXAMPLE_ESP_WIFI_PASS); + } else { + ESP_LOGE(TAG, "UNEXPECTED EVENT"); + } + + ESP_ERROR_CHECK(esp_event_handler_unregister(IP_EVENT, IP_EVENT_STA_GOT_IP, &event_handler)); + ESP_ERROR_CHECK(esp_event_handler_unregister(WIFI_EVENT, ESP_EVENT_ANY_ID, &event_handler)); + vEventGroupDelete(s_wifi_event_group); + return ESP_OK; +} + +#elif ESP_IDF_VERSION_MAJOR < 4 /* event handler for wifi events */ static esp_err_t wifi_event_handler(void *ctx, system_event_t *event) { @@ -270,7 +387,8 @@ int wifi_init_sta(void) int wifi_show_ip(void) { - /* ESP_LOGI(TAG, "got ip:" IPSTR, IP2STR(&event->ip_info.ip)); */ - return 0; + /* TODO Causes panic: ESP_LOGI(TAG, "got ip:" IPSTR, + * IP2STR(&event->ip_info.ip)); */ + return ESP_OK; } #endif diff --git a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/sdkconfig.defaults b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/sdkconfig.defaults index f8bce25f..ff9a5d4c 100644 --- a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/sdkconfig.defaults +++ b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/sdkconfig.defaults @@ -1,26 +1,15 @@ -CONFIG_FREERTOS_HZ=1000 -CONFIG_ESP32_DEFAULT_CPU_FREQ_240=y - -# -# Default main stack size -# -# This is typically way bigger than needed for stack size. See user_settings.h -# -CONFIG_ESP_MAIN_TASK_STACK_SIZE=55500 +# sdkconfig.defaults for ESP8266 + ESP32 -# Legacy stack size for older ESP-IDF versions -CONFIG_MAIN_TASK_STACK_SIZE=55500 +# CONFIG_ESP_PANIC_PRINT_REBOOT is not set +CONFIG_ESP_PANIC_PRINT_REBOOT=n +CONFIG_ESP_PANIC_PRINT_HALT=y -# -# Compiler options -# -CONFIG_COMPILER_OPTIMIZATION_DEFAULT=y -CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_ENABLE=y -CONFIG_COMPILER_OPTIMIZATION_ASSERTION_LEVEL=2 -CONFIG_COMPILER_HIDE_PATHS_MACROS=y -CONFIG_COMPILER_STACK_CHECK_MODE_NORM=y -CONFIG_COMPILER_STACK_CHECK=y +# CONFIG_ESP_HTTP_CLIENT_ENABLE_HTTPS is not set +CONFIG_ESP_HTTP_CLIENT_ENABLE_HTTPS=n +CONFIG_FREERTOS_GLOBAL_DATA_LINK_IRAM=y +CONFIG_HEAP_DISABLE_IRAM=y +CONFIG_FREERTOS_HZ=1000 # # Partition Table # diff --git a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/wolfssl_client_ESP8266.vgdbproj b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/wolfssl_client_ESP8266.vgdbproj new file mode 100644 index 00000000..f0501227 --- /dev/null +++ b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_client/wolfssl_client_ESP8266.vgdbproj @@ -0,0 +1,292 @@ + + + + + + Unknown + + true + + c9687472-a434-43a7-9026-7914f425b9b4 + true + true + SourceDirs + + + + + + com.visualgdb.xtensa-lx106-elf + + 8.4.0 + 8.1 + 1 + + + + release/v3.4 + rtos-sdk/v3.4 + RTOS_SDK + + 0 + + + + + + + + + + + + + Default + + false + + BuildMachine + BuiltinShortcut + + + + + + + COM70 + + 74880 + 8 + None + One + None + + + 0 + false + true + false + ASCII + + + 255 + 0 + 0 + 0 + + + 255 + 169 + 169 + 169 + + + 255 + 211 + 211 + 211 + + + 255 + 144 + 238 + 144 + + + 255 + 169 + 169 + 169 + + + + 16 + true + true + true + true + 0 + + LF + false + false + false + + + + true + + + + + True + + true + true + + Enabled + true + true + true + + true + + + + false + + apiModeling.google.GTest + core.builtin.BuiltinFunctions + core.builtin.NoReturnFunctions + core.CallAndMessage + core.DivideZero + core.DynamicTypePropagation + core.NonnilStringConstants + core.NonNullParamChecker + core.NullDereference + core.StackAddressEscape + core.UndefinedBinaryOperatorResult + core.uninitialized.ArraySubscript + core.uninitialized.Assign + core.uninitialized.Branch + core.uninitialized.CapturedBlockVariable + core.uninitialized.UndefReturn + core.VLASize + cplusplus.NewDelete + cplusplus.NewDeleteLeaks + cplusplus.SelfAssignment + deadcode.DeadStores + nullability.NullPassedToNonnull + nullability.NullReturnedFromNonnull + security.insecureAPI.getpw + security.insecureAPI.gets + security.insecureAPI.mkstemp + security.insecureAPI.mktemp + security.insecureAPI.UncheckedReturn + security.insecureAPI.vfork + unix.API + unix.cstring.BadSizeArg + unix.cstring.NullArg + unix.Malloc + unix.MallocSizeof + unix.MismatchedDeallocator + unix.StdCLibraryFunctions + unix.Vfork + + + -analyzer-store=region + -analyzer-opt-analyze-nested-blocks + -analyzer-eagerly-assume + + + + + + Debug + + build/Debug + sdkconfig-debug + false + + + + Release + + build/Release + sdkconfig-release + false + + + + + + + + + + + + + false + false + false + false + false + false + false + false + false + + false + false + false + false + false + false + true + false + None + false + false + + true + false + false + true + 0 + false + 0 + true + false + + + openocd + + -f interface/ftdi/tigard.cfg -f target/esp8266.cfg + + + + false + + 131072 + Enabled + + set remotetimeout 60 + target remote :$$SYS:GDB_PORT$$ + mon reset halt + load + mon xtensa_no_interrupts_during_steps on + mon esp8266_autofeed_watchdog on + + false + 0 + 0 + false + + 0 + Soft + 4096 + 4096 + + size4M + freq40M + QIO + + + + true + Disabled + 0 + false + false + true + false + false + + _estack + 0 + false + + true + + \ No newline at end of file diff --git a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_server/components/wolfssl/CMakeLists.txt b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_server/components/wolfssl/CMakeLists.txt index e82e19b6..e129a64e 100644 --- a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_server/components/wolfssl/CMakeLists.txt +++ b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_server/components/wolfssl/CMakeLists.txt @@ -82,8 +82,10 @@ function(FIND_WOLFSSL_DIRECTORY OUTPUT_FOUND_WOLFSSL_DIRECTORY) else() get_filename_component(CURRENT_SEARCH_DIR "$ENV{WOLFSSL_ROOT}" ABSOLUTE) IS_WOLFSSL_SOURCE("${CURRENT_SEARCH_DIR}" FOUND_WOLFSSL) - if("${FOUND_WOLFSSL}") - message(STATUS "Found WOLFSSL_ROOT via Environment Variable:") + if( FOUND_WOLFSSL ) + message(STATUS "Found WOLFSSL_ROOT via Environment Variable: ${CURRENT_SEARCH_DIR}") + set(${OUTPUT_FOUND_WOLFSSL_DIRECTORY} ${CURRENT_SEARCH_DIR} PARENT_SCOPE) + return() else() message(FATAL_ERROR "WOLFSSL_ROOT Environment Variable defined, but path not found:") message(STATUS "$ENV{WOLFSSL_ROOT}") diff --git a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_test/components/wolfssl/CMakeLists.txt b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_test/components/wolfssl/CMakeLists.txt index e82e19b6..e129a64e 100644 --- a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_test/components/wolfssl/CMakeLists.txt +++ b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_test/components/wolfssl/CMakeLists.txt @@ -82,8 +82,10 @@ function(FIND_WOLFSSL_DIRECTORY OUTPUT_FOUND_WOLFSSL_DIRECTORY) else() get_filename_component(CURRENT_SEARCH_DIR "$ENV{WOLFSSL_ROOT}" ABSOLUTE) IS_WOLFSSL_SOURCE("${CURRENT_SEARCH_DIR}" FOUND_WOLFSSL) - if("${FOUND_WOLFSSL}") - message(STATUS "Found WOLFSSL_ROOT via Environment Variable:") + if( FOUND_WOLFSSL ) + message(STATUS "Found WOLFSSL_ROOT via Environment Variable: ${CURRENT_SEARCH_DIR}") + set(${OUTPUT_FOUND_WOLFSSL_DIRECTORY} ${CURRENT_SEARCH_DIR} PARENT_SCOPE) + return() else() message(FATAL_ERROR "WOLFSSL_ROOT Environment Variable defined, but path not found:") message(STATUS "$ENV{WOLFSSL_ROOT}") diff --git a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_test/main/main.c b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_test/main/main.c index 1c21bd93..2fd41d3f 100644 --- a/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_test/main/main.c +++ b/extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/examples/wolfssl_test/main/main.c @@ -36,6 +36,10 @@ #include #include +/* set to 0 for one benchmark, +** set to 1 for continuous benchmark loop */ +#define TEST_LOOP 0 + /* ** the wolfssl component can be installed in either: ** @@ -190,7 +194,10 @@ void app_main(void) #if defined(NO_ESP32_CRYPT) ESP_LOGI(TAG, "NO_ESP32_CRYPT defined! HW acceleration DISABLED."); #else - #if defined(CONFIG_IDF_TARGET_ESP32C3) + #if defined(CONFIG_IDF_TARGET_ESP32C2) + ESP_LOGI(TAG, "ESP32_CRYPT is enabled for ESP32-C2."); + + #elif defined(CONFIG_IDF_TARGET_ESP32C3) ESP_LOGI(TAG, "ESP32_CRYPT is enabled for ESP32-C3."); #elif defined(CONFIG_IDF_TARGET_ESP32S2) @@ -239,8 +246,11 @@ void app_main(void) loops++; } - while (ret == 0); - ESP_LOGI(TAG, "loops = %d", loops); + while (TEST_LOOP && (ret == 0)); + +#if defined TEST_LOOP && (TEST_LOOP == 1) + ESP_LOGI(TAG, "Test loops completed: %d", loops); +#endif /* note wolfCrypt_Cleanup() should always be called when finished. ** This is called at the end of wolf_test_task(); @@ -266,8 +276,12 @@ void app_main(void) - (uxTaskGetStackHighWaterMark(NULL))); #endif +#ifdef WOLFSSL_ESPIDF_EXIT_MESSAGE + ESP_LOGI(TAG, WOLFSSL_ESPIDF_EXIT_MESSAGE); +#else ESP_LOGI(TAG, "\n\nDone!\n\n" "If running from idf.py monitor, press twice: Ctrl+]"); +#endif /* done */ while (1) { diff --git a/extra/wolfssl/wolfssl/IDE/Espressif/README.md b/extra/wolfssl/wolfssl/IDE/Espressif/README.md index dea3b36a..5bb1622f 100644 --- a/extra/wolfssl/wolfssl/IDE/Espressif/README.md +++ b/extra/wolfssl/wolfssl/IDE/Espressif/README.md @@ -154,7 +154,48 @@ CMake Error at run_serial_tool.cmake:56 (message): Solution: -Press and hold`EN` button, press and release `IO0` button, then release `EN` button. +Press and hold `EN` button, press and release `IO0` button, then release `EN` button. + +### Unknown CMake command "esptool_py_flash_project_args". + +This unintuitive error was observed when including an unneeded `set(COMPONENTS` in the project-level CMakeLists.txt +and attempting to build with an older toolchain, such as the RTOS SDK 3.4 for the ESP8266. + +### PermissionError: [Errno 13] Permission denied could not open port {} + +This error, other than the obvious permissions, also occurs when the port is in use by another application: + +```text +Traceback (most recent call last): + File "/home/gojimmypi/.espressif/python_env/rtos3.4_py3.10_env/lib/python3.10/site-packages/serial/serialposix.py", line 322, in open + self.fd = os.open(self.portstr, os.O_RDWR | os.O_NOCTTY | os.O_NONBLOCK) +PermissionError: [Errno 13] Permission denied: '/dev/ttyS55' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + [... snip ...] +raise SerialException(msg.errno, "could not open port {}: {}".format(self._port, msg)) +serial.serialutil.SerialException: [Errno 13] could not open port /dev/ttyS55: [Errno 13] Permission denied: '/dev/ttyS55' +``` +### Panic Task watchdog got triggered. + +Long-running code may trip the watchdog timer. + +``` +Task watchdog got triggered. + +Guru Meditation Error: Core 0 panic'ed (unknown). Exception was unhandled. +``` + +The watchdog needs to be [fed](https://docs.espressif.com/projects/esp8266-rtos-sdk/en/latest/api-reference/system/wdts.html?highlight=watchdog#_CPPv418esp_task_wdt_resetv) on a regular basis +with `void esp_task_wdt_reset(void)` from `esp8266/include/esp_task_wdt.h`. + +Try turning off the WDT in menuconfig, or for Makefiles: + +``` +EXTRA_CFLAGS += -DNO_WATCHDOG +``` #### Other Solutions diff --git a/extra/wolfssl/wolfssl/IDE/Espressif/include.am b/extra/wolfssl/wolfssl/IDE/Espressif/include.am index 66203415..5011e51f 100644 --- a/extra/wolfssl/wolfssl/IDE/Espressif/include.am +++ b/extra/wolfssl/wolfssl/IDE/Espressif/include.am @@ -95,6 +95,7 @@ EXTRA_DIST+= IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/include/time_hel EXTRA_DIST+= IDE/Espressif/ESP-IDF/examples/wolfssl_client/main/include/wifi_connect.h EXTRA_DIST+= IDE/Espressif/ESP-IDF/examples/wolfssl_client/VisualGDB/README.md +EXTRA_DIST+= IDE/Espressif/ESP-IDF/examples/wolfssl_client/wolfssl_client_ESP8266.vgdbproj EXTRA_DIST+= IDE/Espressif/ESP-IDF/examples/wolfssl_client/VisualGDB/wolfssl_client_IDF_v5_ESP32.sln EXTRA_DIST+= IDE/Espressif/ESP-IDF/examples/wolfssl_client/VisualGDB/wolfssl_client_IDF_v5_ESP32.vgdbproj diff --git a/extra/wolfssl/wolfssl/IDE/IAR-EWARM/Projects/user_settings.h b/extra/wolfssl/wolfssl/IDE/IAR-EWARM/Projects/user_settings.h index 289a4d71..c6274a29 100644 --- a/extra/wolfssl/wolfssl/IDE/IAR-EWARM/Projects/user_settings.h +++ b/extra/wolfssl/wolfssl/IDE/IAR-EWARM/Projects/user_settings.h @@ -1,4 +1,3 @@ - #define NO_MAIN_DRIVER #define BENCH_EMBEDDED #define NO_WRITEV @@ -17,10 +16,59 @@ #define WOLFSSL_GENSEED_FORTEST /* Warning: define your own seed gen */ -#define TFM_TIMING_RESISTANT +/* A few examples of different math options below. + * + * See examples/configs/user_settings_template.h for a more + * detailed template. */ +#if 1 + /* Use only single precision (SP) math and algorithms. + * SP math is written to accelerate specific/common key + * sizes and curves. This adds code from sp_c32.c, or one of the specific + * assembly implementations like sp_cortexm.c. This code is faster than the + * multi-precision support because it's optimized for the key/curve. + * The SP math can be used together with any multi-precision math library + * if WOLFSSL_SP_MATH is removed. If only standard keys/curves are being + * used the multi-precision math is not required. + */ + #define WOLFSSL_SP_MATH + /* Enable SP ECC support */ + #define WOLFSSL_HAVE_SP_ECC + /* Enable SP RSA support */ + #define WOLFSSL_HAVE_SP_RSA + /* Enable SP DH support */ + #define WOLFSSL_HAVE_SP_DH + /* Reduce stack use specifically in SP implementation. */ + #define WOLFSSL_SP_SMALL_STACK + /* use smaller version of code */ + #define WOLFSSL_SP_SMALL + /* Assembly optimized version - sp_cortexm.c */ + //#define WOLFSSL_SP_ARM_CORTEX_M_ASM +#elif 1 + /* Use SP math for all key sizes and curves. This will use + * the multi-precision (MP) math implementation in sp_int.c */ + #define WOLFSSL_SP_MATH_ALL + /* Disable use of dynamic stack items */ + #define WOLFSSL_SP_NO_DYN_STACK + /* use smaller version of code */ + #define WOLFSSL_SP_SMALL +#elif 1 + /* Fast Math (tfm.c) (stack based and timing resistant) */ + #define USE_FAST_MATH + /* Enable Fast Math Timing Resistance */ + #define TFM_TIMING_RESISTANT +#else + /* Normal (integer.c) (heap based, not timing resistant) - not recommended*/ + #define USE_INTEGER_HEAP_MATH +#endif + +/* Enable ECC Timing Resistance */ #define ECC_TIMING_RESISTANT +/* Enables blinding mode, to prevent timing attacks */ #define WC_RSA_BLINDING +/* reduce stack use. For variables over 100 bytes allocate from heap */ +#define WOLFSSL_SMALL_STACK +/* disable mutex locking */ #define SINGLE_THREADED /* or define RTOS option */ /* #define WOLFSSL_CMSIS_RTOS */ #define NO_FILESYSTEM diff --git a/extra/wolfssl/wolfssl/IDE/IAR-EWARM/embOS/SAMV71_XULT/embOS_wolfcrypt_benchmark_SAMV71_XULT/settings/wolfcrypt_benchmark_Debug.jlink b/extra/wolfssl/wolfssl/IDE/IAR-EWARM/embOS/SAMV71_XULT/embOS_wolfcrypt_benchmark_SAMV71_XULT/settings/wolfcrypt_benchmark_Debug.jlink deleted file mode 100644 index 3a2fb474..00000000 --- a/extra/wolfssl/wolfssl/IDE/IAR-EWARM/embOS/SAMV71_XULT/embOS_wolfcrypt_benchmark_SAMV71_XULT/settings/wolfcrypt_benchmark_Debug.jlink +++ /dev/null @@ -1,39 +0,0 @@ -[BREAKPOINTS] -ForceImpTypeAny = 0 -ShowInfoWin = 1 -EnableFlashBP = 2 -BPDuringExecution = 0 -[CFI] -CFISize = 0x00 -CFIAddr = 0x00 -[CPU] -MonModeVTableAddr = 0xFFFFFFFF -MonModeDebug = 0 -MaxNumAPs = 0 -LowPowerHandlingMode = 0 -OverrideMemMap = 0 -AllowSimulation = 1 -ScriptFile="" -[FLASH] -CacheExcludeSize = 0x00 -CacheExcludeAddr = 0x00 -MinNumBytesFlashDL = 0 -SkipProgOnCRCMatch = 1 -VerifyDownload = 1 -AllowCaching = 1 -EnableFlashDL = 2 -Override = 1 -Device="ATSAMV71Q21" -[GENERAL] -WorkRAMSize = 0x00 -WorkRAMAddr = 0x00 -RAMUsageLimit = 0x00 -[SWO] -SWOLogFile="" -[MEM] -RdOverrideOrMask = 0x00 -RdOverrideAndMask = 0xFFFFFFFF -RdOverrideAddr = 0xFFFFFFFF -WrOverrideOrMask = 0x00 -WrOverrideAndMask = 0xFFFFFFFF -WrOverrideAddr = 0xFFFFFFFF diff --git a/extra/wolfssl/wolfssl/IDE/IAR-EWARM/embOS/SAMV71_XULT/embOS_wolfcrypt_benchmark_SAMV71_XULT/wolfcrypt_benchmark.ewt b/extra/wolfssl/wolfssl/IDE/IAR-EWARM/embOS/SAMV71_XULT/embOS_wolfcrypt_benchmark_SAMV71_XULT/wolfcrypt_benchmark.ewt deleted file mode 100644 index 86013a04..00000000 --- a/extra/wolfssl/wolfssl/IDE/IAR-EWARM/embOS/SAMV71_XULT/embOS_wolfcrypt_benchmark_SAMV71_XULT/wolfcrypt_benchmark.ewt +++ /dev/null @@ -1,2382 +0,0 @@ - - - - 2 - - Debug - - ARM - - 1 - - C-STAT - 259 - - 259 - - 0 - - 1 - 600 - 0 - 2 - 0 - 1 - 100 - - - 1.3.2 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - RuntimeChecking - 0 - - 2 - 1 - 1 - - - - - - - - - - - - - - - - - - - - - - Release - - ARM - - 0 - - C-STAT - 259 - - 259 - - 0 - - 1 - 600 - 0 - 2 - 0 - 1 - 100 - - - 1.3.2 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - RuntimeChecking - 0 - - 2 - 1 - 0 - - - - - - - - - - - - - - - - - - - - - - Application - - $PROJ_DIR$\Application\runBenchmarks.c - - - - benchmark - - $PROJ_DIR$\..\..\..\..\..\wolfcrypt\benchmark\benchmark.c - - - - Device_Support - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\DeviceSupport\startup_sam.c - - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\DeviceSupport\system_sam.c - - - - Setup - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\Setup\BSP.c - - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\Setup\HardFaultHandler.S - - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\Setup\JLINKMEM_Process.c - - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\Setup\OS_Error.c - - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\Setup\RTOSInit_SAMV71_CMSIS.c - - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\Setup\SEGGER_HardFaultHandler.c - - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\Setup\SEGGER_RTT.c - - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\Setup\SEGGER_RTT_printf.c - - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\Setup\SEGGER_SYSVIEW.c - - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\Setup\SEGGER_SYSVIEW_Config_embOS.c - - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\Setup\SEGGER_SYSVIEW_embOS.c - - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\Setup\xmtx.c - - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\Setup\xmtx2.c - - - - - diff --git a/extra/wolfssl/wolfssl/IDE/IAR-EWARM/embOS/SAMV71_XULT/embOS_wolfcrypt_lib_SAMV71_XULT/wolfcrypt_lib.ewt b/extra/wolfssl/wolfssl/IDE/IAR-EWARM/embOS/SAMV71_XULT/embOS_wolfcrypt_lib_SAMV71_XULT/wolfcrypt_lib.ewt deleted file mode 100644 index e0722daa..00000000 --- a/extra/wolfssl/wolfssl/IDE/IAR-EWARM/embOS/SAMV71_XULT/embOS_wolfcrypt_lib_SAMV71_XULT/wolfcrypt_lib.ewt +++ /dev/null @@ -1,2400 +0,0 @@ - - - - 2 - - Debug - - ARM - - 1 - - C-STAT - 259 - - 259 - - 0 - - 1 - 600 - 0 - 2 - 0 - 1 - 100 - - - 1.3.2 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - RuntimeChecking - 0 - - 2 - 1 - 1 - - - - - - - - - - - - - - - - - - - - - - Release - - ARM - - 0 - - C-STAT - 259 - - 259 - - 0 - - 1 - 600 - 0 - 2 - 0 - 1 - 100 - - - 1.3.2 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - RuntimeChecking - 0 - - 2 - 1 - 0 - - - - - - - - - - - - - - - - - - - - - - wolfcrypt_sources - - $PROJ_DIR$\..\..\..\..\..\wolfcrypt\src\aes.c - - - $PROJ_DIR$\..\..\..\..\..\wolfcrypt\src\asn.c - - - $PROJ_DIR$\..\..\..\..\..\wolfcrypt\src\chacha.c - - - $PROJ_DIR$\..\..\..\..\..\wolfcrypt\src\chacha20_poly1305.c - - - $PROJ_DIR$\..\..\..\..\..\wolfcrypt\src\coding.c - - - $PROJ_DIR$\..\..\..\..\..\wolfcrypt\src\des3.c - - - $PROJ_DIR$\..\..\..\..\..\wolfcrypt\src\dh.c - - - $PROJ_DIR$\..\..\..\..\..\wolfcrypt\src\dsa.c - - - $PROJ_DIR$\..\..\..\..\..\wolfcrypt\src\ecc.c - - - $PROJ_DIR$\..\..\..\..\..\wolfcrypt\src\hash.c - - - $PROJ_DIR$\..\..\..\..\..\wolfcrypt\src\kdf.c - - - $PROJ_DIR$\..\..\..\..\..\wolfcrypt\src\hmac.c - - - $PROJ_DIR$\..\..\..\..\..\wolfcrypt\src\md4.c - - - $PROJ_DIR$\..\..\..\..\..\wolfcrypt\src\md5.c - - - $PROJ_DIR$\..\..\..\..\..\wolfcrypt\src\memory.c - - - $PROJ_DIR$\..\..\..\..\..\wolfcrypt\src\misc.c - - - $PROJ_DIR$\..\..\..\..\..\wolfcrypt\src\poly1305.c - - - $PROJ_DIR$\..\..\..\..\..\wolfcrypt\src\pwdbased.c - - - $PROJ_DIR$\..\..\..\..\..\wolfcrypt\src\random.c - - - $PROJ_DIR$\..\..\..\..\..\wolfcrypt\src\rsa.c - - - $PROJ_DIR$\..\..\..\..\..\wolfcrypt\src\sha.c - - - $PROJ_DIR$\..\..\..\..\..\wolfcrypt\src\sha256.c - - - $PROJ_DIR$\..\..\..\..\..\wolfcrypt\src\sha512.c - - - $PROJ_DIR$\..\..\..\..\..\wolfcrypt\src\tfm.c - - - $PROJ_DIR$\..\..\..\..\..\wolfcrypt\src\wc_encrypt.c - - - $PROJ_DIR$\..\..\..\..\..\wolfcrypt\src\wc_port.c - - - - - diff --git a/extra/wolfssl/wolfssl/IDE/IAR-EWARM/embOS/SAMV71_XULT/embOS_wolfcrypt_test_SAMV71_XULT/settings/wolfcrypt_test_Debug.jlink b/extra/wolfssl/wolfssl/IDE/IAR-EWARM/embOS/SAMV71_XULT/embOS_wolfcrypt_test_SAMV71_XULT/settings/wolfcrypt_test_Debug.jlink deleted file mode 100644 index 3a2fb474..00000000 --- a/extra/wolfssl/wolfssl/IDE/IAR-EWARM/embOS/SAMV71_XULT/embOS_wolfcrypt_test_SAMV71_XULT/settings/wolfcrypt_test_Debug.jlink +++ /dev/null @@ -1,39 +0,0 @@ -[BREAKPOINTS] -ForceImpTypeAny = 0 -ShowInfoWin = 1 -EnableFlashBP = 2 -BPDuringExecution = 0 -[CFI] -CFISize = 0x00 -CFIAddr = 0x00 -[CPU] -MonModeVTableAddr = 0xFFFFFFFF -MonModeDebug = 0 -MaxNumAPs = 0 -LowPowerHandlingMode = 0 -OverrideMemMap = 0 -AllowSimulation = 1 -ScriptFile="" -[FLASH] -CacheExcludeSize = 0x00 -CacheExcludeAddr = 0x00 -MinNumBytesFlashDL = 0 -SkipProgOnCRCMatch = 1 -VerifyDownload = 1 -AllowCaching = 1 -EnableFlashDL = 2 -Override = 1 -Device="ATSAMV71Q21" -[GENERAL] -WorkRAMSize = 0x00 -WorkRAMAddr = 0x00 -RAMUsageLimit = 0x00 -[SWO] -SWOLogFile="" -[MEM] -RdOverrideOrMask = 0x00 -RdOverrideAndMask = 0xFFFFFFFF -RdOverrideAddr = 0xFFFFFFFF -WrOverrideOrMask = 0x00 -WrOverrideAndMask = 0xFFFFFFFF -WrOverrideAddr = 0xFFFFFFFF diff --git a/extra/wolfssl/wolfssl/IDE/IAR-EWARM/embOS/SAMV71_XULT/embOS_wolfcrypt_test_SAMV71_XULT/wolfcrypt_test.ewt b/extra/wolfssl/wolfssl/IDE/IAR-EWARM/embOS/SAMV71_XULT/embOS_wolfcrypt_test_SAMV71_XULT/wolfcrypt_test.ewt deleted file mode 100644 index 2847ab53..00000000 --- a/extra/wolfssl/wolfssl/IDE/IAR-EWARM/embOS/SAMV71_XULT/embOS_wolfcrypt_test_SAMV71_XULT/wolfcrypt_test.ewt +++ /dev/null @@ -1,2382 +0,0 @@ - - - - 2 - - Debug - - ARM - - 1 - - C-STAT - 259 - - 259 - - 0 - - 1 - 600 - 0 - 2 - 0 - 1 - 100 - - - 1.3.2 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - RuntimeChecking - 0 - - 2 - 1 - 1 - - - - - - - - - - - - - - - - - - - - - - Release - - ARM - - 0 - - C-STAT - 259 - - 259 - - 0 - - 1 - 600 - 0 - 2 - 0 - 1 - 100 - - - 1.3.2 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - RuntimeChecking - 0 - - 2 - 1 - 0 - - - - - - - - - - - - - - - - - - - - - - Application - - $PROJ_DIR$\Application\runWolfcryptTests.c - - - - Device_Support - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\DeviceSupport\startup_sam.c - - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\DeviceSupport\system_sam.c - - - - Setup - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\Setup\BSP.c - - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\Setup\HardFaultHandler.S - - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\Setup\JLINKMEM_Process.c - - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\Setup\OS_Error.c - - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\Setup\RTOSInit_SAMV71_CMSIS.c - - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\Setup\SEGGER_HardFaultHandler.c - - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\Setup\SEGGER_RTT.c - - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\Setup\SEGGER_RTT_printf.c - - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\Setup\SEGGER_SYSVIEW.c - - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\Setup\SEGGER_SYSVIEW_Config_embOS.c - - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\Setup\SEGGER_SYSVIEW_embOS.c - - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\Setup\xmtx.c - - - $PROJ_DIR$\..\..\extract_trial_here\Start\BoardSupport\Atmel\SAMV71_XPlainedUltra\Setup\xmtx2.c - - - - wolfcrypt_test - - $PROJ_DIR$\..\..\..\..\..\wolfcrypt\test\test.c - - - - - diff --git a/extra/wolfssl/wolfssl/IDE/LINUX-SGX/sgx_t_static.mk b/extra/wolfssl/wolfssl/IDE/LINUX-SGX/sgx_t_static.mk index 5f26391b..ebb718cf 100644 --- a/extra/wolfssl/wolfssl/IDE/LINUX-SGX/sgx_t_static.mk +++ b/extra/wolfssl/wolfssl/IDE/LINUX-SGX/sgx_t_static.mk @@ -88,6 +88,7 @@ Wolfssl_C_Files :=$(WOLFSSL_ROOT)/wolfcrypt/src/aes.c\ $(WOLFSSL_ROOT)/wolfcrypt/src/signature.c\ $(WOLFSSL_ROOT)/wolfcrypt/src/sp_c32.c\ $(WOLFSSL_ROOT)/wolfcrypt/src/sp_c64.c\ + $(WOLFSSL_ROOT)/wolfcrypt/src/sp_int.c\ $(WOLFSSL_ROOT)/src/ssl.c\ $(WOLFSSL_ROOT)/src/tls.c\ $(WOLFSSL_ROOT)/wolfcrypt/src/wc_encrypt.c\ diff --git a/extra/wolfssl/wolfssl/IDE/MPLABX16/README.md b/extra/wolfssl/wolfssl/IDE/MPLABX16/README.md new file mode 100644 index 00000000..a35b6dec --- /dev/null +++ b/extra/wolfssl/wolfssl/IDE/MPLABX16/README.md @@ -0,0 +1,62 @@ +# wolfSSL MPLAB X Project Files for XC16 + +This directory contains project files for the Microchip MPLAB X IDE. These +projects have been set up to use the Microchip PIC24 Starter Kit +and the Microchip XC16 compiler. + +In order to generate the necessary auto-generated MPLAB X files, make sure +to import the wolfssl.X project into your MPLAB X workspace before trying to +build the wolfCrypt test. This will correctly set up the respective project's +Makefiles. + +## Included Project Files + +### wolfSSL library (wolfssl.X) + +This project builds a static wolfSSL library. The settings for this project are in `user_settings.h`: +``` +/IDE/MPLABX16/user_settings.h +``` + +After this project has been built, the compiled library will be located at: +``` +/IDE/MPLABX16/wolfssl.X/dist/default/production/wolfssl.X.a +``` + +### wolfCrypt Test App (wolfcrypt_test.X) + +This project tests the wolfCrypt cryptography modules. It is generally a good +idea to run this first on an embedded system after compiling wolfSSL in order +to verify all underlying crypto is working correctly. This project depends on +files generated by Microchip's MCC tool to view the UART output. Follow the +steps below to generate that code. + +## Generating MCC UART code + +1. Open the MPLAB Code Configurator application. + +2. Set the Project path to the wolfSSL/IDE/MPLABX16 and enter your PIC device +into the interface. + +3. Select MCC Clasic as the content type and click `Finish`. + +4. Under the Device Resources section, find the UART entry and add the UART1 +peripheral. + +5. Note the UART settings and check the `Enable UART Interrupts` and +`Redirect Printf to UART` boxes. + +6. Click the `Generate` button. + + +**Note** : If using an older version of `xc16`, you may have to add the +following to `user_settings.h`. +``` +#define WOLFSSL_HAVE_MIN +#define WOLFSSL_HAVE_MAX +``` + +## Support + +Please send questions or comments to support@wolfssl.com + diff --git a/extra/wolfssl/wolfssl/IDE/MPLABX16/include.am b/extra/wolfssl/wolfssl/IDE/MPLABX16/include.am new file mode 100644 index 00000000..d7f855b1 --- /dev/null +++ b/extra/wolfssl/wolfssl/IDE/MPLABX16/include.am @@ -0,0 +1,8 @@ +# vim:ft=automake +# All paths should be given relative to the root +# + +EXTRA_DIST += \ + IDE/MPLABX16/README.md \ + IDE/MPLABX16/main.c \ + IDE/MPLABX16/user_settings.h diff --git a/extra/wolfssl/wolfssl/IDE/MPLABX16/main.c b/extra/wolfssl/wolfssl/IDE/MPLABX16/main.c new file mode 100644 index 00000000..fadad81d --- /dev/null +++ b/extra/wolfssl/wolfssl/IDE/MPLABX16/main.c @@ -0,0 +1,39 @@ +/* main.c + * + * Copyright (C) 2006-2024 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +#include + +#include +#include + +#include + +#include "xc.h" +#include "mcc_generated_files/mcc.h" + +int main(void) { + SYSTEM_Initialize(); + + wolfcrypt_test(NULL); + + return 0; +} + diff --git a/extra/wolfssl/wolfssl/IDE/MPLABX16/user_settings.h b/extra/wolfssl/wolfssl/IDE/MPLABX16/user_settings.h new file mode 100644 index 00000000..d4754db8 --- /dev/null +++ b/extra/wolfssl/wolfssl/IDE/MPLABX16/user_settings.h @@ -0,0 +1,414 @@ +/* Example custom user settings for wolfSSL */ + +#ifndef WOLFSSL_USER_SETTINGS_H +#define WOLFSSL_USER_SETTINGS_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* ------------------------------------------------------------------------- */ +/* Platform */ +/* ------------------------------------------------------------------------- */ +#undef WOLFSSL_GENERAL_ALIGNMENT +#define WOLFSSL_GENERAL_ALIGNMENT 4 + +#undef SINGLE_THREADED +#define SINGLE_THREADED + +#undef WOLFSSL_SMALL_STACK +#define WOLFSSL_SMALL_STACK + +#define MICROCHIP_PIC24 + +/* Define for older versions of xc16 */ +#if 0 + #define WOLFSSL_HAVE_MIN + #define WOLFSSL_HAVE_MAX +#endif + +#ifdef MICROCHIP_PIC24 + #define SIZEOF_LONG_LONG 8 + #define SIZEOF_LONG 4 + #define SINGLE_THREADED + #define WOLFSSL_USER_IO + #define NO_WRITEV + #define NO_DEV_RANDOM + #define NO_FILESYSTEM + #define BENCH_EMBEDDED + #define WC_16BIT_CPU + #define WORD64_AVAILABLE + #define WOLFSSL_GENSEED_FORTEST +#endif + +/* ------------------------------------------------------------------------- */ +/* Math Configuration */ +/* ------------------------------------------------------------------------- */ +#if 1 + #undef USE_FAST_MATH + #define USE_FAST_MATH + + #undef FP_MAX_BITS + #define FP_MAX_BITS 2048 +#else + #define WOLFSSL_SP_MATH + #define WOLFSSL_SP_SMALL + #define WOLFSSL_SP_MATH_ALL + #define SP_INT_BITS 256 +#endif + + +#ifdef USE_FAST_MATH + #undef TFM_TIMING_RESISTANT + #define TFM_TIMING_RESISTANT + + /* Optimizations */ + //#define TFM_MIPS +#endif + +/* ------------------------------------------------------------------------- */ +/* Crypto */ +/* ------------------------------------------------------------------------- */ +/* ECC */ +#if 1 + #undef HAVE_ECC + #define HAVE_ECC + + /* Manually define enabled curves */ + #undef ECC_USER_CURVES + #define ECC_USER_CURVES + + /* Reduces heap usage, but slower */ + #undef ECC_TIMING_RESISTANT + #define ECC_TIMING_RESISTANT + + //#define HAVE_ECC192 + //#define HAVE_ECC224 + //#define HAVE_ECC384 + /* Fixed point cache (speeds repeated operations against same private key) */ +#if 1 + #undef FP_ECC + #define FP_ECC + #ifdef FP_ECC + /* Bits / Entries */ + #undef FP_ENTRIES + #define FP_ENTRIES 2 + #undef FP_LUT + #define FP_LUT 4 + #endif + /* Optional ECC calculation method */ + /* Note: doubles heap usage, but slightly faster */ + #undef ECC_SHAMIR + #define ECC_SHAMIR + + + #ifdef USE_FAST_MATH + /* use reduced size math buffers for ecc points */ + #undef ALT_ECC_SIZE + #define ALT_ECC_SIZE + + /* Enable TFM optimizations for ECC */ + #if defined(HAVE_ECC192) || defined(HAVE_ALL_CURVES) + #define TFM_ECC192 + #endif + #if defined(HAVE_ECC224) || defined(HAVE_ALL_CURVES) + #define TFM_ECC224 + #endif + #if !defined(NO_ECC256) || defined(HAVE_ALL_CURVES) + #define TFM_ECC256 + #endif + #if defined(HAVE_ECC384) || defined(HAVE_ALL_CURVES) + #define TFM_ECC384 + #endif + #if defined(HAVE_ECC521) || defined(HAVE_ALL_CURVES) + #define TFM_ECC521 + #endif + #endif +#endif +#endif + +/* RSA */ +#undef NO_RSA +#if 0 + /* half as much memory but twice as slow */ + #undef RSA_LOW_MEM +#define RSA_LOW_MEM + + #undef WC_RSA_PSS + #define WC_RSA_PSS + + /* timing resistance */ + #undef WC_RSA_BLINDING + #define WC_RSA_BLINDING +#else + #define NO_RSA +#endif + +/* AES */ +#undef NO_AES +#if 1 + #undef HAVE_AES_DECRYPT + #define HAVE_AES_DECRYPT + + #undef HAVE_AESGCM + #define HAVE_AESGCM + + /* GCM Method: GCM_SMALL, GCM_WORD32 or GCM_TABLE */ + #undef GCM_SMALL + #define GCM_SMALL + + /* #undef HAVE_AESCCM + #define HAVE_AESCCM */ + + /* #undef WOLFSSL_AES_DIRECT + #define WOLFSSL_AES_DIRECT */ + + #undef NO_AES_CBC + #define NO_AES_CBC +#else + #define NO_AES +#endif + +/* DES3 */ +#undef NO_DES3 +#if 0 + #undef WOLFSSL_DES_ECB + #define WOLFSSL_DES_ECB +#else + #define NO_DES3 +#endif + + +/* ChaCha20 / Poly1305 */ +#undef HAVE_CHACHA +#undef HAVE_POLY1305 +#if 0 + #define HAVE_CHACHA + #define HAVE_POLY1305 + + /* Needed for Poly1305 */ + #undef HAVE_ONE_TIME_AUTH + #define HAVE_ONE_TIME_AUTH +#endif + +/* Ed25519 / Curve25519 */ +#undef HAVE_CURVE25519 +#undef HAVE_ED25519 +#if 0 + #define HAVE_CURVE25519 + #define HAVE_ED25519 + + /* Optionally use small math (less flash usage, but much slower) */ + #if 0 + #define CURVED25519_SMALL + #endif +#endif + + +/* ------------------------------------------------------------------------- */ +/* Hashing */ +/* ------------------------------------------------------------------------- */ +/* Sha */ +#undef NO_SHA +#if 0 + /* 1k smaller, but 25% slower */ + #define USE_SLOW_SHA +#else + #define NO_SHA +#endif + +/* Sha256 */ +#undef NO_SHA256 +#if 1 +#else + #define NO_SHA256 +#endif + +/* Sha512 */ +#undef WOLFSSL_SHA512 +#if 0 + #define WOLFSSL_SHA512 + + /* Sha384 */ + #undef WOLFSSL_SHA384 + #if 0 + #define WOLFSSL_SHA384 + #endif + + /* over twice as small, but 50% slower */ + #define USE_SLOW_SHA2 +#endif + + +/* ------------------------------------------------------------------------- */ +/* Benchmark / Test */ +/* ------------------------------------------------------------------------- */ +/* Use reduced benchmark / test sizes */ +#undef BENCH_EMBEDDED +#define BENCH_EMBEDDED + +//#undef USE_CERT_BUFFERS_2048 +//#define USE_CERT_BUFFERS_2048 + +#undef USE_CERT_BUFFERS_1024 +#define USE_CERT_BUFFERS_1024 + +#undef USE_CERT_BUFFERS_256 +#define USE_CERT_BUFFERS_256 + + +/* ------------------------------------------------------------------------- */ +/* Time */ +/* ------------------------------------------------------------------------- */ +#if 0 + /* Override Current Time */ + /* Allows custom "custom_time()" function to be used for benchmark */ + #define WOLFSSL_USER_CURRTIME + #define USER_TICKS + extern unsigned long custom_time(unsigned long* timer); + #define XTIME custom_time +#else + //#warning Time/RTC disabled + #undef NO_ASN_TIME + #define NO_ASN_TIME +#endif + +/* ------------------------------------------------------------------------- */ +/* Debugging */ +/* ------------------------------------------------------------------------- */ +#undef DEBUG_WOLFSSL + +#if 0 + #define DEBUG_WOLFSSL + #define WOLFSSL_DEBUG_TLS + /* Use this to measure / print heap usage */ + #undef USE_WOLFSSL_MEMORY + #define USE_WOLFSSL_MEMORY + #undef WOLFSSL_TRACK_MEMORY + #define WOLFSSL_TRACK_MEMORY +#else + #undef NO_WOLFSSL_MEMORY + #define NO_WOLFSSL_MEMORY +#endif + +/* ------------------------------------------------------------------------- */ +/* Misc */ +/* ------------------------------------------------------------------------- */ +#define WOLFSSL_ASN_TEMPLATE +#define NO_ERROR_STRINGS +#define NO_LARGE_HASH_TEST +#define NO_PKCS12 +#define NO_PKCS8 +#define WOLFSSL_NO_PEM + + +/* ------------------------------------------------------------------------- */ +/* Enable Features */ +/* ------------------------------------------------------------------------- */ +#undef KEEP_PEER_CERT +#define KEEP_PEER_CERT + +#undef HAVE_COMP_KEY +#define HAVE_COMP_KEY + +#undef WOLFSSL_TLS13 +#define WOLFSSL_TLS13 + +#undef HAVE_HKDF +#define HAVE_HKDF + +#undef HAVE_TLS_EXTENSIONS +#define HAVE_TLS_EXTENSIONS + +#ifdef HAVE_ECC +#undef HAVE_SUPPORTED_CURVES +#define HAVE_SUPPORTED_CURVES +#endif + +#undef WOLFSSL_BASE64_ENCODE +#define WOLFSSL_BASE64_ENCODE + +/* TLS Session Cache */ +#if 0 + #define SMALL_SESSION_CACHE +#else + #define NO_SESSION_CACHE +#endif + + +/* ------------------------------------------------------------------------- */ +/* Disable Features */ +/* ------------------------------------------------------------------------- */ +#undef NO_WOLFSSL_SERVER +//#define NO_WOLFSSL_SERVER + +#undef NO_WOLFSSL_CLIENT +#define NO_WOLFSSL_CLIENT + +#undef NO_CRYPT_TEST +//#define NO_CRYPT_TEST + +#undef NO_CRYPT_BENCHMARK +//#define NO_CRYPT_BENCHMARK + +/* In-lining of misc.c functions */ +/* If defined, must include wolfcrypt/src/misc.c in build */ +/* Slower, but about 1k smaller */ +#undef NO_INLINE +#define NO_INLINE + +#undef NO_FILESYSTEM +#define NO_FILESYSTEM + +#undef NO_WRITEV +#define NO_WRITEV + +#undef NO_MAIN_DRIVER +#define NO_MAIN_DRIVER + +#undef NO_DEV_RANDOM +#define NO_DEV_RANDOM + +#undef NO_PSK +#define NO_PSK + +#undef NO_DSA +#define NO_DSA + +#undef NO_DH +#define NO_DH + +#undef NO_RC4 +#define NO_RC4 + +#undef NO_OLD_TLS +#define NO_OLD_TLS + +#undef WOLFSSL_NO_TLS12 +#define WOLFSSL_NO_TLS12 + +#undef NO_PSK +//#define NO_PSK +#define WOLFSSL_STATIC_PSK + +#undef NO_MD4 +#define NO_MD4 + +#undef NO_PWDBASED +#define NO_PWDBASED + +#undef NO_MD5 +#define NO_MD5 + +#undef NO_DES3 +#define NO_DES3 + +#undef NO_CODING +//#define NO_CODING + + +#ifdef __cplusplus +} +#endif + +#endif /* WOLFSSL_USER_SETTINGS_H */ diff --git a/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfcrypt_test.X/Makefile b/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfcrypt_test.X/Makefile new file mode 100644 index 00000000..fca8e2cc --- /dev/null +++ b/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfcrypt_test.X/Makefile @@ -0,0 +1,113 @@ +# +# There exist several targets which are by default empty and which can be +# used for execution of your targets. These targets are usually executed +# before and after some main targets. They are: +# +# .build-pre: called before 'build' target +# .build-post: called after 'build' target +# .clean-pre: called before 'clean' target +# .clean-post: called after 'clean' target +# .clobber-pre: called before 'clobber' target +# .clobber-post: called after 'clobber' target +# .all-pre: called before 'all' target +# .all-post: called after 'all' target +# .help-pre: called before 'help' target +# .help-post: called after 'help' target +# +# Targets beginning with '.' are not intended to be called on their own. +# +# Main targets can be executed directly, and they are: +# +# build build a specific configuration +# clean remove built files from a configuration +# clobber remove all built files +# all build all configurations +# help print help mesage +# +# Targets .build-impl, .clean-impl, .clobber-impl, .all-impl, and +# .help-impl are implemented in nbproject/makefile-impl.mk. +# +# Available make variables: +# +# CND_BASEDIR base directory for relative paths +# CND_DISTDIR default top distribution directory (build artifacts) +# CND_BUILDDIR default top build directory (object files, ...) +# CONF name of current configuration +# CND_ARTIFACT_DIR_${CONF} directory of build artifact (current configuration) +# CND_ARTIFACT_NAME_${CONF} name of build artifact (current configuration) +# CND_ARTIFACT_PATH_${CONF} path to build artifact (current configuration) +# CND_PACKAGE_DIR_${CONF} directory of package (current configuration) +# CND_PACKAGE_NAME_${CONF} name of package (current configuration) +# CND_PACKAGE_PATH_${CONF} path to package (current configuration) +# +# NOCDDL + + +# Environment +MKDIR=mkdir +CP=cp +CCADMIN=CCadmin +RANLIB=ranlib + + +# build +build: .build-post + +.build-pre: +# Add your pre 'build' code here... + +.build-post: .build-impl +# Add your post 'build' code here... + + +# clean +clean: .clean-post + +.clean-pre: +# Add your pre 'clean' code here... +# WARNING: the IDE does not call this target since it takes a long time to +# simply run make. Instead, the IDE removes the configuration directories +# under build and dist directly without calling make. +# This target is left here so people can do a clean when running a clean +# outside the IDE. + +.clean-post: .clean-impl +# Add your post 'clean' code here... + + +# clobber +clobber: .clobber-post + +.clobber-pre: +# Add your pre 'clobber' code here... + +.clobber-post: .clobber-impl +# Add your post 'clobber' code here... + + +# all +all: .all-post + +.all-pre: +# Add your pre 'all' code here... + +.all-post: .all-impl +# Add your post 'all' code here... + + +# help +help: .help-post + +.help-pre: +# Add your pre 'help' code here... + +.help-post: .help-impl +# Add your post 'help' code here... + + + +# include project implementation makefile +include nbproject/Makefile-impl.mk + +# include project make variables +include nbproject/Makefile-variables.mk diff --git a/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfcrypt_test.X/nbproject/configurations.xml b/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfcrypt_test.X/nbproject/configurations.xml new file mode 100755 index 00000000..fe95c871 --- /dev/null +++ b/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfcrypt_test.X/nbproject/configurations.xml @@ -0,0 +1,285 @@ + + + + + ../user_settings.h + + + + + + ../mcc_generated_files/clock.c + ../mcc_generated_files/interrupt_manager.c + ../mcc_generated_files/mcc.c + ../mcc_generated_files/pin_manager.c + ../mcc_generated_files/system.c + ../mcc_generated_files/traps.c + ../mcc_generated_files/uart1.c + + ../../../wolfcrypt/test/test.c + ../main.c + + + Makefile + + + + .. + ../../wolfcrypt/test + ../../../wolfcrypt/test + + Makefile + + + + localhost + PIC24FJ1024GB610 + + + PKOBSKDEPlatformTool + XC16 + 2.10 + 4 + + + + + + + + + + + + + + + + + + + false + true + + + + + + + false + false + + false + + false + false + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfcrypt_test.X/nbproject/include.am b/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfcrypt_test.X/nbproject/include.am new file mode 100644 index 00000000..ab8cb715 --- /dev/null +++ b/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfcrypt_test.X/nbproject/include.am @@ -0,0 +1,8 @@ +j vim:ft=automake +# All paths should be given relative to the root +# + +EXTRA_DIST += \ + IDE/MPLABX16/wolfcrypt_test.X/nbproject/configurations.xml \ + IDE/MPLABX16/wolfcrypt_test.X/nbproject/project.xml + diff --git a/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfcrypt_test.X/nbproject/private/configurations.xml b/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfcrypt_test.X/nbproject/private/configurations.xml new file mode 100644 index 00000000..ba95e245 --- /dev/null +++ b/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfcrypt_test.X/nbproject/private/configurations.xml @@ -0,0 +1,25 @@ + + + Makefile + 0 + + + + /Applications/microchip/xc16/v2.10/bin + + place holder 1 + place holder 2 + + + + + true + 0 + 0 + 0 + + + + + + diff --git a/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfcrypt_test.X/nbproject/private/private.xml b/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfcrypt_test.X/nbproject/private/private.xml new file mode 100644 index 00000000..6807a2ba --- /dev/null +++ b/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfcrypt_test.X/nbproject/private/private.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfcrypt_test.X/nbproject/project.xml b/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfcrypt_test.X/nbproject/project.xml new file mode 100755 index 00000000..358f3c77 --- /dev/null +++ b/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfcrypt_test.X/nbproject/project.xml @@ -0,0 +1,33 @@ + + + com.microchip.mplab.nbide.embedded.makeproject + + + wolfcrypt_test + b34c4937-7042-4352-88b1-7717bcdf8aeb + 0 + c + + h + ISO-8859-1 + + + ../wolfssl.X + + + .. + ../../wolfcrypt/test + ../../../wolfcrypt/test + + + + default + 2 + + + + false + + + + diff --git a/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfssl.X/Makefile b/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfssl.X/Makefile new file mode 100644 index 00000000..fca8e2cc --- /dev/null +++ b/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfssl.X/Makefile @@ -0,0 +1,113 @@ +# +# There exist several targets which are by default empty and which can be +# used for execution of your targets. These targets are usually executed +# before and after some main targets. They are: +# +# .build-pre: called before 'build' target +# .build-post: called after 'build' target +# .clean-pre: called before 'clean' target +# .clean-post: called after 'clean' target +# .clobber-pre: called before 'clobber' target +# .clobber-post: called after 'clobber' target +# .all-pre: called before 'all' target +# .all-post: called after 'all' target +# .help-pre: called before 'help' target +# .help-post: called after 'help' target +# +# Targets beginning with '.' are not intended to be called on their own. +# +# Main targets can be executed directly, and they are: +# +# build build a specific configuration +# clean remove built files from a configuration +# clobber remove all built files +# all build all configurations +# help print help mesage +# +# Targets .build-impl, .clean-impl, .clobber-impl, .all-impl, and +# .help-impl are implemented in nbproject/makefile-impl.mk. +# +# Available make variables: +# +# CND_BASEDIR base directory for relative paths +# CND_DISTDIR default top distribution directory (build artifacts) +# CND_BUILDDIR default top build directory (object files, ...) +# CONF name of current configuration +# CND_ARTIFACT_DIR_${CONF} directory of build artifact (current configuration) +# CND_ARTIFACT_NAME_${CONF} name of build artifact (current configuration) +# CND_ARTIFACT_PATH_${CONF} path to build artifact (current configuration) +# CND_PACKAGE_DIR_${CONF} directory of package (current configuration) +# CND_PACKAGE_NAME_${CONF} name of package (current configuration) +# CND_PACKAGE_PATH_${CONF} path to package (current configuration) +# +# NOCDDL + + +# Environment +MKDIR=mkdir +CP=cp +CCADMIN=CCadmin +RANLIB=ranlib + + +# build +build: .build-post + +.build-pre: +# Add your pre 'build' code here... + +.build-post: .build-impl +# Add your post 'build' code here... + + +# clean +clean: .clean-post + +.clean-pre: +# Add your pre 'clean' code here... +# WARNING: the IDE does not call this target since it takes a long time to +# simply run make. Instead, the IDE removes the configuration directories +# under build and dist directly without calling make. +# This target is left here so people can do a clean when running a clean +# outside the IDE. + +.clean-post: .clean-impl +# Add your post 'clean' code here... + + +# clobber +clobber: .clobber-post + +.clobber-pre: +# Add your pre 'clobber' code here... + +.clobber-post: .clobber-impl +# Add your post 'clobber' code here... + + +# all +all: .all-post + +.all-pre: +# Add your pre 'all' code here... + +.all-post: .all-impl +# Add your post 'all' code here... + + +# help +help: .help-post + +.help-pre: +# Add your pre 'help' code here... + +.help-post: .help-impl +# Add your post 'help' code here... + + + +# include project implementation makefile +include nbproject/Makefile-impl.mk + +# include project make variables +include nbproject/Makefile-variables.mk diff --git a/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfssl.X/nbproject/configurations.xml b/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfssl.X/nbproject/configurations.xml new file mode 100644 index 00000000..bd423ae2 --- /dev/null +++ b/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfssl.X/nbproject/configurations.xml @@ -0,0 +1,290 @@ + + + + + ../user_settings.h + + + + + + ../../../wolfcrypt/src/aes.c + ../../../wolfcrypt/src/arc4.c + ../../../wolfcrypt/src/asm.c + ../../../wolfcrypt/src/asn.c + ../../../wolfcrypt/src/blake2b.c + ../../../wolfcrypt/src/blake2s.c + ../../../wolfcrypt/src/camellia.c + ../../../wolfcrypt/src/chacha.c + ../../../wolfcrypt/src/chacha20_poly1305.c + ../../../wolfcrypt/src/cmac.c + ../../../wolfcrypt/src/coding.c + ../../../wolfcrypt/src/compress.c + ../../../wolfcrypt/src/cpuid.c + ../../../wolfcrypt/src/cryptocb.c + ../../../wolfcrypt/src/curve25519.c + ../../../wolfcrypt/src/curve448.c + ../../../wolfcrypt/src/des3.c + ../../../wolfcrypt/src/dh.c + ../../../wolfcrypt/src/dilithium.c + ../../../wolfcrypt/src/dsa.c + ../../../wolfcrypt/src/ecc.c + ../../../wolfcrypt/src/ecc_fp.c + ../../../wolfcrypt/src/eccsi.c + ../../../wolfcrypt/src/ed25519.c + ../../../wolfcrypt/src/ed448.c + ../../../wolfcrypt/src/error.c + ../../../wolfcrypt/src/evp.c + ../../../wolfcrypt/src/ext_kyber.c + ../../../wolfcrypt/src/falcon.c + ../../../wolfcrypt/src/fe_448.c + ../../../wolfcrypt/src/fe_low_mem.c + ../../../wolfcrypt/src/fe_operations.c + ../../../wolfcrypt/src/ge_448.c + ../../../wolfcrypt/src/ge_low_mem.c + ../../../wolfcrypt/src/ge_operations.c + ../../../wolfcrypt/src/hash.c + ../../../wolfcrypt/src/hmac.c + ../../../wolfcrypt/src/hpke.c + ../../../wolfcrypt/src/integer.c + ../../../wolfcrypt/src/kdf.c + ../../../wolfcrypt/src/logging.c + ../../../wolfcrypt/src/md2.c + ../../../wolfcrypt/src/md4.c + ../../../wolfcrypt/src/md5.c + ../../../wolfcrypt/src/memory.c + ../../../wolfcrypt/src/misc.c + ../../../wolfcrypt/src/pkcs12.c + ../../../wolfcrypt/src/pkcs7.c + ../../../wolfcrypt/src/poly1305.c + ../../../wolfcrypt/src/pwdbased.c + ../../../wolfcrypt/src/random.c + ../../../wolfcrypt/src/rc2.c + ../../../wolfcrypt/src/ripemd.c + ../../../wolfcrypt/src/rsa.c + ../../../wolfcrypt/src/sakke.c + ../../../wolfcrypt/src/sha.c + ../../../wolfcrypt/src/sha256.c + ../../../wolfcrypt/src/sha3.c + ../../../wolfcrypt/src/sha512.c + ../../../wolfcrypt/src/signature.c + ../../../wolfcrypt/src/siphash.c + ../../../wolfcrypt/src/sp_arm32.c + ../../../wolfcrypt/src/sp_arm64.c + ../../../wolfcrypt/src/sp_armthumb.c + ../../../wolfcrypt/src/sp_c32.c + ../../../wolfcrypt/src/sp_c64.c + ../../../wolfcrypt/src/sp_int.c + ../../../wolfcrypt/src/sphincs.c + ../../../wolfcrypt/src/srp.c + ../../../wolfcrypt/src/tfm.c + ../../../wolfcrypt/src/wc_encrypt.c + ../../../wolfcrypt/src/wc_pkcs11.c + ../../../wolfcrypt/src/wc_port.c + ../../../wolfcrypt/src/wolfevent.c + ../../../wolfcrypt/src/wolfmath.c + + + ../../../src/crl.c + ../../../src/internal.c + ../../../src/keys.c + ../../../src/ssl.c + ../../../src/tls.c + ../../../src/tls13.c + ../../../src/wolfio.c + + + + Makefile + + + + .. + ../../src + ../../wolfcrypt/src + ../../wolfcrypt/test + ../../../src + ../../../wolfcrypt/src + + Makefile + + + + localhost + PIC24FJ1024GB610 + + + noID + XC16 + 2.10 + 4 + + + + + + + + + + + + + + + false + false + + + + + + + false + false + + false + + false + false + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfssl.X/nbproject/include.am b/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfssl.X/nbproject/include.am new file mode 100644 index 00000000..14d44b06 --- /dev/null +++ b/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfssl.X/nbproject/include.am @@ -0,0 +1,7 @@ +# vim:ft=automake +# All paths should be given relative to the root +# + +EXTRA_DIST += \ + IDE/MPLABX16/wolfssl.X/nbproject/configurations.xml \ + IDE/MPLABX16/wolfssl.X/nbproject/project.xml diff --git a/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfssl.X/nbproject/project.xml b/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfssl.X/nbproject/project.xml new file mode 100644 index 00000000..1fb81b58 --- /dev/null +++ b/extra/wolfssl/wolfssl/IDE/MPLABX16/wolfssl.X/nbproject/project.xml @@ -0,0 +1,34 @@ + + + com.microchip.mplab.nbide.embedded.makeproject + + + wolfssl + 93bbfc3a-a0fa-4d48-bbc8-6cd47a2bd05b + 0 + c + + h + ISO-8859-1 + + + + .. + ../../src + ../../wolfcrypt/src + ../../wolfcrypt/test + ../../../src + ../../../wolfcrypt/src + + + + default + 3 + + + + false + + + + diff --git a/extra/wolfssl/wolfssl/IDE/MSVS-2019-AZSPHERE/shared/util.h b/extra/wolfssl/wolfssl/IDE/MSVS-2019-AZSPHERE/shared/util.h index 005676f4..fc54a709 100644 --- a/extra/wolfssl/wolfssl/IDE/MSVS-2019-AZSPHERE/shared/util.h +++ b/extra/wolfssl/wolfssl/IDE/MSVS-2019-AZSPHERE/shared/util.h @@ -26,6 +26,7 @@ #include #include #include +#include #define _GNU_SOURCE /* defines NI_NUMERICHOST */ #ifndef NI_MAXHOST diff --git a/extra/wolfssl/wolfssl/IDE/Renesas/e2studio/RA6M4/common/user_settings.h b/extra/wolfssl/wolfssl/IDE/Renesas/e2studio/RA6M4/common/user_settings.h index 4263164e..48541531 100644 --- a/extra/wolfssl/wolfssl/IDE/Renesas/e2studio/RA6M4/common/user_settings.h +++ b/extra/wolfssl/wolfssl/IDE/Renesas/e2studio/RA6M4/common/user_settings.h @@ -115,4 +115,5 @@ #endif #define CUSTOM_RAND_GENERATE_BLOCK wc_fspsm_GenerateRandBlock - +/* use original asn parsing */ +#define WOLFSSL_ASN_ORIGINAL diff --git a/extra/wolfssl/wolfssl/IDE/Renesas/e2studio/RA6M4/test/.cproject b/extra/wolfssl/wolfssl/IDE/Renesas/e2studio/RA6M4/test/.cproject index 61375953..e7bb1ceb 100644 --- a/extra/wolfssl/wolfssl/IDE/Renesas/e2studio/RA6M4/test/.cproject +++ b/extra/wolfssl/wolfssl/IDE/Renesas/e2studio/RA6M4/test/.cproject @@ -94,7 +94,7 @@ - + @@ -103,7 +103,7 @@ - +