Merge pull request #54 from 1nfiniteloop/x509-uplift-reviewed

Updated x509 modules to match openssl 1.1.0h API, reviewed

Author: CyberShadow

CONTRIBUTING.md

@@ -0,0 +1,69 @@
+# Summary
+
+This instruction describes how to pragmatically upgrade the openssl d-bindings
+to match the C-headers version, in this case current version is 1.1.0h.
+
+## Steps
+
+### Preparation
+
+1. Get d-step tool with:
+    ```
+    curl \
+        --location \
+        https://github.com/jacob-carlborg/dstep/releases/download/v1.0.0/dstep-1.0.0-linux-x86_64.tar.xz \
+        | tar -x --xz --location /usr/local/bin
+    ```
+   See more @ <https://github.com/jacob-carlborg/dstep/releases>.
+2. Clone openssl with `git clone https://github.com/openssl/openssl` and
+   checkout correct tag, example `cd openssl && git checkout OpenSSL_1_1_0h`.
+3. Headers with suffix ".h.in" need to be parsed to .h before generation,
+   Configure openssl with `./configure` and build generated file with 
+   `make build_generated`.
+
+### Check dependencies
+
+A good approach is to convert the files in ascending order for no of
+dependencies. Example when converting x509, use order: `x509_vfy.h` which
+is used by 1 other file, `x509v3.h` is used by 3 other files, `x509.h`
+is used by 11 other files.
+
+```
+grep -r 'include <openssl/x509_vfy.h>$' C/
+grep -r 'include <openssl/x509v3.h>$' C/
+grep -r 'include <openssl/x509.h>$' C/
+```
+
+### Generate module from C header
+
+1. Generate d-module from the openssl c-header with
+   `dstep --space-after-function-name=false -Iinclude/ include/openssl/<file>`. Commit the change.
+
+### Manual patching
+
+Below is a checklist for common known issues which needs manual work:
+
+1. d-step doesn't resolve includes. Translate "import" statements from
+   `#include` in header-file accordingly, and possible check in the old .d-file
+   for special cases.
+2. Function aliases in C-headers without argument list, example
+   `#define alias-name function` are generated as enum types. This gives
+   compilation error similar to "missing argument for parameter #1".
+   Replace "enum" with "alias" accordingly.
+3. Many struct definitions is removed, instead a declaration ia added into
+   `ossl_typ.d`, Example `grep -r 'struct X509_pubkey_st' C/` shows that struct
+   definition is removed from `x509.h` and instead a declaration is added in
+   `ossl_typ.h`. Other types might be removed, check the header-file and adjust
+   accordingly if the type is missing when compiling.
+4. Check the header-file for "ifdef|ifndef", search for "OPENSSL_*" where some
+   statements has historically been translated into "version" in d-modules.
+5. Macros `STACK_OF`, `DEFINE_STACK_OF`: in version 1.1.0h the macro `STACK_OF`
+   in `safestack.d` has changed. During generation it's properly expanded into
+   a type prefixed with `stack_st`. Since other dependent modules might not be
+   uplifted, a declaration sometimes need to be inserted to make it
+   compile. It will result in "type missing "stack_st_...". Check in which
+   header the macro `DEFINE_STACK_OF(<type>)` is defined in and manually add
+   `struct stack_st_<type-name>` to make it compile. However these functions 
+   will not work properly during linkage until `safestack.d` is uplifted,
+   see macro `DEFINE_STACK_OF` in safestack.h.
+   

deimos/openssl/asn1.d

@@ -820,6 +820,8 @@ ASN1_OBJECT* 	d2i_ASN1_OBJECT(ASN1_OBJECT** a,const(ubyte)** pp,
 mixin(DECLARE_ASN1_ITEM!"ASN1_OBJECT");
 
 /+mixin DECLARE_STACK_OF!(ASN1_OBJECT);+/
+struct stack_st_ASN1_OBJECT;  // define type to make it compile, needed until this module and safestack.d is converted and supports "DEFINE_STACK_OF"
+
 mixin DECLARE_ASN1_SET_OF!(ASN1_OBJECT);
 
 ASN1_STRING* 	ASN1_STRING_new();

deimos/openssl/conf.d

@@ -78,6 +78,8 @@ struct CONF_VALUE
 	char* value;
 	}
 
+struct lhash_st_CONF_VALUE;     // declare type to make it compile, needed until this module and lhash.d is uplifted to >= 1.1.0h and supports "DEFINE_LHASH_OF"
+
 /+mixin DECLARE_STACK_OF!(CONF_VALUE);+/
 mixin DECLARE_LHASH_OF!(CONF_VALUE);
 

deimos/openssl/crypto.d

@@ -142,6 +142,9 @@ public import deimos.openssl.symhacks;
 extern (C):
 nothrow:
 
+alias OPENSSL_buf2hexstr = char* function(const ubyte*, long);
+alias OPENSSL_hexstr2buf = ubyte* function(const char*, long*);
+
 /* Backward compatibility to SSLeay */
 /* This is more to be used to check the correct DLL is being used
  * in the MS world. */

deimos/openssl/ossl_typ.d

@@ -171,17 +171,38 @@ alias ecdsa_method ECDSA_METHOD;
 
 import deimos.openssl.x509;
 import deimos.openssl.x509_vfy;
+
+struct ssl_dane_st;
+alias SSL_DANE = ssl_dane_st;
+
+struct x509_st;
 alias x509_st X509;
 alias X509_algor_st X509_ALGOR;
+struct X509_crl_st;
 alias X509_crl_st X509_CRL;
 struct x509_crl_method_st;
 alias x509_crl_method_st X509_CRL_METHOD;
+struct x509_revoked_st;
 alias x509_revoked_st X509_REVOKED;
+struct X509_name_st;
 alias X509_name_st X509_NAME;
+struct X509_pubkey_st;
 alias X509_pubkey_st X509_PUBKEY;
+struct x509_store_st;
 alias x509_store_st X509_STORE;
-/*struct x509_store_ctx_st;*/
+struct x509_store_ctx_st;
 alias x509_store_ctx_st X509_STORE_CTX;
+struct x509_lookup_st;
+alias X509_LOOKUP = x509_lookup_st;
+
+struct x509_object_st;
+alias X509_OBJECT = x509_object_st;
+struct x509_lookup_method_st;
+alias X509_LOOKUP_METHOD = x509_lookup_method_st;
+struct X509_VERIFY_PARAM_st;
+alias X509_VERIFY_PARAM = X509_VERIFY_PARAM_st;
+
+struct pkcs8_priv_key_info_st;
 alias pkcs8_priv_key_info_st PKCS8_PRIV_KEY_INFO;
 
 import deimos.openssl.x509v3;

deimos/openssl/pem.d

@@ -477,8 +477,6 @@ mixin(DECLARE_PEM_rw!("X509", "X509")());
 
 mixin(DECLARE_PEM_rw!("X509_AUX", "X509")());
 
-mixin(DECLARE_PEM_rw!("X509_CERT_PAIR", "X509_CERT_PAIR")());
-
 mixin(DECLARE_PEM_rw!("X509_REQ", "X509_REQ")());
 mixin(DECLARE_PEM_write!("X509_REQ_NEW", "X509_REQ")());
 

deimos/openssl/safestack.d

@@ -126,6 +126,8 @@ mixin DECLARE_SPECIAL_STACK_OF!(OPENSSL_STRING, char);
  * nul-terminated. These should also be distinguished from "normal"
  * stacks. */
 
+struct stack_st_OPENSSL_STRING;  // declare type to make it compile, needed until this module is uplifted to >= 1.1.0h and supports "DECLARE_SPECIAL_STACK_OF"
+
 alias void* OPENSSL_BLOCK;
 mixin DECLARE_SPECIAL_STACK_OF!(OPENSSL_BLOCK, void);
 /* SKM_sk_... stack macros are internal to safestack.h: