lib: wrap crypt API to hide errno usage

In order to prevent using the global errno, replace it with a static
version and create a wrapper function which returns the error value.

Signed-off-by: Steffen Jaeckel <jaeckel-floss@eyet-services.de>
Reviewed-by: Simon Glass <sjg@chromium.org>
Reviewed-by: Heiko Schocher <hs@denx.de>

include/crypt.h

@@ -9,5 +9,6 @@
  * @equal       Pointer to an int where the result is stored
  *                 '0' = unequal
  *                 '1' = equal
+ * @return 0 on success, error code of errno else
  */
-void crypt_compare(const char *should, const char *passphrase, int *equal);
+int crypt_compare(const char *should, const char *passphrase, int *equal);

lib/crypt/alg-sha256.h

@@ -1,12 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0+ */
 /* Copyright (C) 2020 Steffen Jaeckel <jaeckel-floss@eyet-services.de> */
 
-#ifndef USE_HOSTCC
-#include "common.h"
-#else
-#include <string.h>
-#endif
-
 #include "u-boot/sha256.h"
 
 #define INCLUDE_sha256crypt 1

lib/crypt/alg-sha512.h

@@ -1,12 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0+ */
 /* Copyright (C) 2020 Steffen Jaeckel <jaeckel-floss@eyet-services.de> */
 
-#ifndef USE_HOSTCC
-#include "common.h"
-#else
-#include <string.h>
-#endif
-
 #include "u-boot/sha512.h"
 
 #define INCLUDE_sha512crypt 1

lib/crypt/crypt-port.h

@@ -18,11 +18,13 @@ extern const unsigned char ascii64[65];
 
 #define b64t ((const char *)ascii64)
 
-void crypt_sha256crypt_rn(const char *phrase, size_t phr_size,
+int crypt_sha256crypt_rn_wrapped(const char *phrase, size_t phr_size,
-			  const char *setting, size_t ARG_UNUSED(set_size),
+				 const char *setting,
-			  uint8_t *output, size_t out_size, void *scratch,
+				 size_t ARG_UNUSED(set_size), uint8_t *output,
-			  size_t scr_size);
+				 size_t out_size, void *scratch,
-void crypt_sha512crypt_rn(const char *phrase, size_t phr_size,
+				 size_t scr_size);
-			  const char *setting, size_t ARG_UNUSED(set_size),
+int crypt_sha512crypt_rn_wrapped(const char *phrase, size_t phr_size,
-			  uint8_t *output, size_t out_size, void *scratch,
+				 const char *setting,
-			  size_t scr_size);
+				 size_t ARG_UNUSED(set_size), uint8_t *output,
+				 size_t out_size, void *scratch,
+				 size_t scr_size);

lib/crypt/crypt-sha256.c

@@ -1,10 +1,14 @@
+// SPDX-License-Identifier: CC0-1.0
+/* Based on libxcrypt v4.4.17-0-g6b110bc */
 /* One way encryption based on the SHA256-based Unix crypt implementation.
  *
  * Written by Ulrich Drepper <drepper at redhat.com> in 2007 [1].
  * Modified by Zack Weinberg <zackw at panix.com> in 2017, 2018.
  * Composed by Björn Esser <besser82 at fedoraproject.org> in 2018.
  * Modified by Björn Esser <besser82 at fedoraproject.org> in 2020.
- * Modified by Steffen Jaeckel <jaeckel-floss at eyet-services.de> in 2020.
+ * Modified by Steffen Jaeckel <jaeckel-floss at eyet-services.de> in 2021
+ * for U-Boot, instead of using the global errno to use a static one
+ * inside this file.
  * To the extent possible under law, the named authors have waived all
  * copyright and related or neighboring rights to this work.
  *
@@ -20,7 +24,7 @@
 #include "crypt-port.h"
 #include "alg-sha256.h"
 
-#include <errno.h>
+#include <linux/errno.h>
 #include <stdio.h>
 #include <stdlib.h>
 
@@ -69,6 +73,25 @@ static_assert (sizeof (struct sha256_buffer) <= ALG_SPECIFIC_SIZE,
                "ALG_SPECIFIC_SIZE is too small for SHA256");
 
 
+/* Use this instead of including errno.h */
+static int errno;
+
+void crypt_sha256crypt_rn(const char *phrase, size_t phr_size,
+			  const char *setting, size_t ARG_UNUSED(set_size),
+			  uint8_t *output, size_t out_size, void *scratch,
+			  size_t scr_size);
+
+int crypt_sha256crypt_rn_wrapped(const char *phrase, size_t phr_size,
+				 const char *setting, size_t set_size,
+				 u8 *output, size_t out_size, void *scratch,
+				 size_t scr_size)
+{
+	errno = 0;
+	crypt_sha256crypt_rn(phrase, phr_size, setting, set_size, output,
+			     out_size, scratch, scr_size);
+	return -errno;
+}
+
 /* Feed CTX with LEN bytes of a virtual byte sequence consisting of
    BLOCK repeated over and over indefinitely.  */
 static void

lib/crypt/crypt-sha512.c

@@ -1,10 +1,14 @@
+// SPDX-License-Identifier: CC0-1.0
+/* Based on libxcrypt v4.4.17-0-g6b110bc */
 /* One way encryption based on the SHA512-based Unix crypt implementation.
  *
  * Written by Ulrich Drepper <drepper at redhat.com> in 2007 [1].
  * Modified by Zack Weinberg <zackw at panix.com> in 2017, 2018.
  * Composed by Björn Esser <besser82 at fedoraproject.org> in 2018.
  * Modified by Björn Esser <besser82 at fedoraproject.org> in 2020.
- * Modified by Steffen Jaeckel <jaeckel-floss at eyet-services.de> in 2020.
+ * Modified by Steffen Jaeckel <jaeckel-floss at eyet-services.de> in 2021
+ * for U-Boot, instead of using the global errno to use a static one
+ * inside this file.
  * To the extent possible under law, the named authors have waived all
  * copyright and related or neighboring rights to this work.
  *
@@ -20,7 +24,7 @@
 #include "crypt-port.h"
 #include "alg-sha512.h"
 
-#include <errno.h>
+#include <linux/errno.h>
 #include <stdio.h>
 #include <stdlib.h>
 
@@ -69,6 +73,25 @@ static_assert (sizeof (struct sha512_buffer) <= ALG_SPECIFIC_SIZE,
                "ALG_SPECIFIC_SIZE is too small for SHA512");
 
 
+/* Use this instead of including errno.h */
+static int errno;
+
+void crypt_sha512crypt_rn(const char *phrase, size_t phr_size,
+			  const char *setting, size_t ARG_UNUSED(set_size),
+			  uint8_t *output, size_t out_size, void *scratch,
+			  size_t scr_size);
+
+int crypt_sha512crypt_rn_wrapped(const char *phrase, size_t phr_size,
+				 const char *setting, size_t set_size,
+				 u8 *output, size_t out_size, void *scratch,
+				 size_t scr_size)
+{
+	errno = 0;
+	crypt_sha512crypt_rn(phrase, phr_size, setting, set_size, output,
+			     out_size, scratch, scr_size);
+	return -errno;
+}
+
 /* Subroutine of _xcrypt_crypt_sha512crypt_rn: Feed CTX with LEN bytes of a
    virtual byte sequence consisting of BLOCK repeated over and over
    indefinitely.  */

lib/crypt/crypt.c

@@ -5,8 +5,8 @@
 #include <crypt.h>
 #include "crypt-port.h"
 
-typedef void (*crypt_fn)(const char *, size_t, const char *, size_t, uint8_t *,
+typedef int (*crypt_fn)(const char *, size_t, const char *, size_t, uint8_t *,
-			 size_t, void *, size_t);
+			size_t, void *, size_t);
 
 const unsigned char ascii64[65] =
 	"./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
@@ -29,19 +29,20 @@ static void equals_constant_time(const void *a_, const void *b_, size_t len,
 	*equal = ret ^ 1;
 }
 
-void crypt_compare(const char *should, const char *passphrase, int *equal)
+int crypt_compare(const char *should, const char *passphrase, int *equal)
 {
 	u8 output[CRYPT_OUTPUT_SIZE], scratch[ALG_SPECIFIC_SIZE];
 	size_t n;
+	int err;
 	struct {
 		const char *prefix;
 		crypt_fn crypt;
 	} crypt_algos[] = {
 #if defined(CONFIG_CRYPT_PW_SHA256)
-		{ "$5$", crypt_sha256crypt_rn },
+		{ "$5$", crypt_sha256crypt_rn_wrapped },
 #endif
 #if defined(CONFIG_CRYPT_PW_SHA512)
-		{ "$6$", crypt_sha512crypt_rn },
+		{ "$6$", crypt_sha512crypt_rn_wrapped },
 #endif
 		{ NULL, NULL }
 	};
@@ -56,18 +57,20 @@ void crypt_compare(const char *should, const char *passphrase, int *equal)
 	}
 
 	if (n >= ARRAY_SIZE(crypt_algos))
-		return;
+		return -EINVAL;
-
-	crypt_algos[n].crypt(passphrase, strlen(passphrase), should, 0, output,
-			     sizeof(output), scratch, sizeof(scratch));
 
+	err = crypt_algos[n].crypt(passphrase, strlen(passphrase), should, 0,
+				   output, sizeof(output), scratch,
+				   sizeof(scratch));
 	/* early return on error, nothing really happened inside the crypt() function */
-	if (errno == ERANGE || errno == EINVAL)
+	if (err)
-		return;
+		return err;
 
 	equals_constant_time(should, output, strlen((const char *)output),
 			     equal);
 
 	memset(scratch, 0, sizeof(scratch));
 	memset(output, 0, sizeof(output));
+
+	return 0;
 }

test/lib/test_crypt.c

@@ -21,19 +21,39 @@
 static int lib_crypt(struct unit_test_state *uts)
 {
 	int equals = 0;
+	int err;
+
+	err = crypt_compare("", "password", &equals);
+	ut_assertf(err != 0, "crypt_compare successful but should not\n");
+	ut_assertf(equals != 1,
+		   "crypt_compare password hash matched but should not\n");
 
 	if (IS_ENABLED(CONFIG_CRYPT_PW_SHA256)) {
-		crypt_compare(
+		err = crypt_compare("$5$", "password", &equals);
+		ut_assertf(err == 0, "crypt-sha256 not successful\n");
+		ut_assertf(
+			equals != 1,
+			"crypt-sha256 password hash matched but should not\n");
+
+		err = crypt_compare(
 			"$5$rounds=640000$TM4lL4zXDG7F4aRX$JM7a9wmvodnA0WasjTztj6mxg.KVuk6doQ/eBhdcapB",
 			"password", &equals);
+		ut_assertf(err == 0, "crypt-sha256 failed: %d\n", err);
 		ut_assertf(equals == 1,
 			   "crypt-sha256 password hash didn't match\n");
 	}
 	equals = 0;
 	if (IS_ENABLED(CONFIG_CRYPT_PW_SHA512)) {
-		crypt_compare(
+		err = crypt_compare("$6$", "password", &equals);
+		ut_assertf(err == 0, "crypt-sha512 not successful\n");
+		ut_assertf(
+			equals != 1,
+			"crypt-sha512 password hash matched but should not\n");
+
+		err = crypt_compare(
 			"$6$rounds=640000$fCTP1F0N5JLq2eND$z5EzK5KZJA9JnOaj5d1Gg/2v6VqFOQJ3bVekWuCPauabutBt/8qzV1exJnytUyhbq3H0bSBXtodwNbtGEi/Tm/",
 			"password", &equals);
+		ut_assertf(err == 0, "crypt-sha512 failed: %d\n", err);
 		ut_assertf(equals == 1,
 			   "crypt-sha512 password hash didn't match\n");
 	}