From: Gerhard Rieger <gerhard@dest-unreach.org>
Date: Mon, 12 Dec 2016 17:58:59 +0000 (+0100)
Subject: Changes to make socat compile with OpenSSL 1.1
X-Git-Tag: tag-1.7.3.2~24
X-Git-Url: http://repo.or.cz/socat.git/commitdiff_plain/008229cb9fe60646fb04b64d3704485c32ce821b?hp=0061ca13345f803ce74a3d53001d05199fc6becd

Changes to make socat compile with OpenSSL 1.1
---

diff --git a/CHANGES b/CHANGES
index 4ee2a76..332f9a3 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,3 +1,11 @@
+####################### V 2.0.0-b10:
+
+porting:
+	Changes to make socat compile with OpenSSL 1.1. 
+	Thanks to Sebastian Andrzej Siewior e.a. from the Debian team for
+	providing the base patch.
+	Debian Bug#828550
+
 
 ####################### V 2.0.0-b9:
 
diff --git a/config.h.in b/config.h.in
index d60b08e..2c2468b 100644
--- a/config.h.in
+++ b/config.h.in
@@ -447,6 +447,15 @@
 #undef HAVE_DTLSv1_client_method
 #undef HAVE_DTLSv1_server_method
 
+/* Define if you have the OpenSSL RAND_egd function */
+#undef HAVE_RAND_egd
+
+/* Define if you have the OpenSSL DH_set0_pqg function */
+#undef HAVE_DH_set0_pqg
+
+/* Define if you have the OpenSSL ASN1_STRING_get0_data function */
+#undef HAVE_ASN1_STRING_get0_data
+
 
 /* Define if you have the flock function */
 #undef HAVE_FLOCK
diff --git a/configure.in b/configure.in
index 03fb8e1..a32473b 100644
--- a/configure.in
+++ b/configure.in
@@ -1467,6 +1467,9 @@ AC_CHECK_FUNC(TLSv1_2_client_method, AC_
 AC_CHECK_FUNC(TLSv1_2_server_method, AC_DEFINE(HAVE_TLSv1_2_server_method), AC_CHECK_LIB(crypt, TLSv1_2_server_method, [LIBS=-lcrypt $LIBS]))
 AC_CHECK_FUNC(DTLSv1_client_method, AC_DEFINE(HAVE_DTLSv1_client_method), AC_CHECK_LIB(crypt, DTLSv1_client_method, [LIBS=-lcrypt $LIBS]))
 AC_CHECK_FUNC(DTLSv1_server_method, AC_DEFINE(HAVE_DTLSv1_server_method), AC_CHECK_LIB(crypt, DTLSv1_server_method, [LIBS=-lcrypt $LIBS]))
+AC_CHECK_FUNC(RAND_egd, AC_DEFINE(HAVE_RAND_egd), AC_CHECK_LIB(crypt, RAND_egd, [LIBS=-lcrypt $LIBS]))
+AC_CHECK_FUNC(DH_set0_pqg, AC_DEFINE(HAVE_DH_set0_pqg), AC_CHECK_LIB(crypt, DH_set0_pqg, [LIBS=-lcrypt $LIBS]))
+AC_CHECK_FUNC(ASN1_STRING_get0_data, AC_DEFINE(HAVE_ASN1_STRING_get0_data), AC_CHECK_LIB(crypt, ASN1_STRING_get0_data, [LIBS=-lcrypt $LIBS]))
 
 dnl Run time checks
 
diff --git a/sslcls.c b/sslcls.c
index 4ec054e..1b54265 100644
--- a/sslcls.c
+++ b/sslcls.c
@@ -339,6 +339,7 @@ void sycSSL_free(SSL *ssl) {
    return;
 }
 
+#if !defined(OPENSSL_NO_EGD) && HAVE_RAND_egd
 int sycRAND_egd(const char *path) {
    int result;
    Debug1("RAND_egd(\"%s\")", path);
@@ -346,6 +347,7 @@ int sycRAND_egd(const char *path) {
    Debug1("RAND_egd() -> %d", result);
    return result;
 }
+#endif
 
 DH *sycPEM_read_bio_DHparams(BIO *bp, DH **x, pem_password_cb *cb, void *u) {
    DH *result;
diff --git a/xio-openssl.c b/xio-openssl.c
index 89a9a9a..2ea0542 100644
--- a/xio-openssl.c
+++ b/xio-openssl.c
@@ -878,7 +878,11 @@ int
    }
 
    if (opt_egd) {
+#if !defined(OPENSSL_NO_EGD) && HAVE_RAND_egd
       sycRAND_egd(opt_egd);
+#else
+      Debug("RAND_egd() is not available by OpenSSL");
+#endif
    }
 
    if (opt_pseudo) {
@@ -1073,35 +1073,48 @@ int
 	 0x02,
       };
       DH *dh;
+      BIGNUM *p = NULL, *g = NULL;
       unsigned long err;
 
-      if ((dh = DH_new()) == NULL) {
-	 while (err = ERR_get_error()) {
-	    Warn1("DH_new(): %s",
-		  ERR_error_string(err, NULL));
-	 }
-	 Error("DH_new() failed");
-      } else {
-	 dh->p = BN_bin2bn(dh2048_p, sizeof(dh2048_p), NULL);
-	 dh->g = BN_bin2bn(dh2048_g, sizeof(dh2048_g), NULL);
-	 if ((dh->p == NULL) || (dh->g == NULL)) {
-	    while (err = ERR_get_error()) {
-	       Warn1("BN_bin2bn(): %s",
-		     ERR_error_string(err, NULL));
-	    }
-	    Error("BN_bin2bn() failed");
-	 } else {
-	    if (sycSSL_CTX_set_tmp_dh(*ctx, dh) <= 0) {
-	       while (err = ERR_get_error()) {
-		  Warn3("SSL_CTX_set_tmp_dh(%p, %p): %s", *ctx, dh,
-			ERR_error_string(err, NULL));
-	       }
-	       Error2("SSL_CTX_set_tmp_dh(%p, %p) failed", *ctx, dh);
-	    }
-	    /*! OPENSSL_free(dh->p,g)? doc does not tell so */
-	 }
-	 DH_free(dh);
+      dh = DH_new();
+      p = BN_bin2bn(dh2048_p, sizeof(dh2048_p), NULL);
+      g = BN_bin2bn(dh2048_g, sizeof(dh2048_g), NULL);
+      if (!dh || !p || !g) {
+         if (dh)
+            DH_free(dh);
+         if (p)
+            BN_free(p);
+         if (g)
+            BN_free(g);
+         while (err = ERR_get_error()) {
+            Warn1("dh2048 setup(): %s",
+                  ERR_error_string(err, NULL));
+         }
+         Error("dh2048 setup failed");
+         goto cont_out;
+      }
+#if HAVE_DH_set0_pqg
+      if (!DH_set0_pqg(dh, p, NULL, g)) {
+	      DH_free(dh);
+	      BN_free(p);
+	      BN_free(g);
+	      goto cont_out;
       }
+#else
+      dh->p = p;
+      dh->g = g;
+#endif /* HAVE_DH_set0_pqg */
+      if (sycSSL_CTX_set_tmp_dh(*ctx, dh) <= 0) {
+         while (err = ERR_get_error()) {
+            Warn3("SSL_CTX_set_tmp_dh(%p, %p): %s", *ctx, dh,
+                  ERR_error_string(err, NULL));
+         }
+         Error2("SSL_CTX_set_tmp_dh(%p, %p) failed", *ctx, dh);
+       }
+      /* p & g are freed by DH_free() once attached */
+      DH_free(dh);
+cont_out:
+      ;
    }
 
 #if defined(EC_KEY)  /* not on Openindiana 5.11 */
@@ -1103,7 +1120,11 @@ static int openssl_SSL_ERROR_SSL(int level, const char *funcname) {
    while (e = ERR_get_error()) {
       Debug1("ERR_get_error(): %lx", e);
       if (e == ((ERR_LIB_RAND<<24)|
+#if defined(RAND_F_RAND_BYTES)
+		(RAND_F_RAND_BYTES<<12)|
+#else
 		(RAND_F_SSLEAY_RAND_BYTES<<12)|
+#endif
 		(RAND_R_PRNG_NOT_SEEDED)) /*0x24064064*/) {
 	 Error("too few entropy; use options \"egd\" or \"pseudo\"");
 	 stat = STAT_NORETRY;
@@ -1396,7 +1408,7 @@ static bool openssl_check_peername(X509_
    int ind = -1;
    X509_NAME_ENTRY *entry;
    ASN1_STRING *data;
-   unsigned char *text;
+   const unsigned char *text;
    ind = X509_NAME_get_index_by_NID(name, NID_commonName, -1);
    if (ind < 0) {
       Info("no COMMONNAME field in peer certificate"); 
@@ -1314,7 +1339,11 @@ static bool openssl_check_peername(X509_NAME *name, const char *peername) {
    }
    entry = X509_NAME_get_entry(name, ind);
    data = X509_NAME_ENTRY_get_data(entry);
+#if HAVE_ASN1_STRING_get0_data
+   text = ASN1_STRING_get0_data(data);
+#else
    text = ASN1_STRING_data(data);
+#endif
    return openssl_check_name((const char *)text, peername);
 }
 
@@ -1454,13 +1466,17 @@ static int openssl_setenv_cert_fields(co
       X509_NAME_ENTRY *entry;
       ASN1_OBJECT *obj;
       ASN1_STRING *data;
-      unsigned char *text;
+      const unsigned char *text;
       int nid;
       entry = X509_NAME_get_entry(name, i);
       obj  = X509_NAME_ENTRY_get_object(entry);
       data = X509_NAME_ENTRY_get_data(entry);
       nid  = OBJ_obj2nid(obj);
+#if HAVE_ASN1_STRING_get0_data
+      text = ASN1_STRING_get0_data(data);
+#else
       text = ASN1_STRING_data(data);
+#endif
       Debug3("SSL peer cert %s entry: %s=\"%s\"", (field[0]?field:"subject"), OBJ_nid2ln(nid), text);
       if (field != NULL && field[0] != '\0') {
 	 xiosetenv3("OPENSSL_X509", field, OBJ_nid2ln(nid), (const char *)text, 2, " // ");