[PATCH v3 0/6] staging: rtl8723bs: explicitly type variables in rtw_security.c

[PATCH v3 0/6] staging: rtl8723bs: explicitly type variables in rtw_security.c

[Pranav Desai]

This v3 series focuses exclusively on explicitly typing the non-standard
'uint' variables in rtw_security.c.

Per Greg's feedback on v2, I have split the explicit sizing of variables
into 6 granular, logical commits to ensure rigorous bisectability and
ease of review:

- Patch 1: Matches packet lengths to struct definitions.
- Patch 2: Converts integer flags to standard booleans and updates 1/0 assignments.
- Patch 3: Explicitly sizes frame types to u8 based on bitmasks.
- Patches 4-6: Explicitly types computed variables, iterators, and helper
  lengths to safely handle math operations.

Changes in v3:
- Split rtw_security.c changes into 6 logical patches (one concept per patch).
- Replaced 1/0 assignments with true/false for boolean flags.
- Dropped hal_intf.c from this series entirely (will address separately).

Link: https://lore.kernel.org/linux-staging/20261006200751.819332-1-contact.pranavdesai@gmail.com/ 

Pranav Desai (6):
  staging: rtl8723bs: use explicit types for lengths in rtw_security.c
  staging: rtl8723bs: use bool for flags in rtw_security.c
  staging: rtl8723bs: use u8 for frame types in rtw_security.c
  staging: rtl8723bs: use explicit types for math variables and
    iterators
  staging: rtl8723bs: use u32 for payload_length in rtw_security.c
  staging: rtl8723bs: use u32 for length variables in rtw_BIP_verify()

 drivers/staging/rtl8723bs/core/rtw_security.c | 68 ++++++++++---------
 1 file changed, 37 insertions(+), 31 deletions(-)

-- 
2.55.0

[PATCH v3 1/6] staging: rtl8723bs: use explicit types for lengths in rtw_security.c

[Pranav Desai]

Replace 'uint' with explicit types for length variables in rtw_security.c
based on how they are used:

- aes_cipher(): use u16 for hdrlen and u32 for plen to match the fields
  in struct pkt_attrib.
- aes_decipher(): use u8 for hdrlen to match struct rx_pkt_attrib. Use
  u32 for plen since it is calculated from a 32-bit packet length.

Signed-off-by: Pranav Desai <contact.pranavdesai@gmail.com>
---
 drivers/staging/rtl8723bs/core/rtw_security.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/drivers/staging/rtl8723bs/core/rtw_security.c b/drivers/staging/rtl8723bs/core/rtw_security.c
index 72af46391e23..af461971c827 100644
--- a/drivers/staging/rtl8723bs/core/rtw_security.c
+++ b/drivers/staging/rtl8723bs/core/rtw_security.c
@@ -780,8 +780,8 @@ static void construct_ctr_preload(u8 *ctr_preload,
 	ctr_preload[15] = (unsigned char)(c % 256);
 }
 
-static signed int aes_cipher(u8 *key, uint hdrlen,
-			     u8 *pframe, uint plen)
+static signed int aes_cipher(u8 *key, u16 hdrlen,
+			     u8 *pframe, u32 plen)
 {
 	uint	qc_exists, a4_exists, i, j, payload_remainder,
 		num_blocks, payload_index;
@@ -980,8 +980,8 @@ u32 rtw_aes_encrypt(struct adapter *padapter, u8 *pxmitframe)
 	return res;
 }
 
-static signed int aes_decipher(u8 *key, uint hdrlen,
-			       u8 *pframe, uint plen)
+static signed int aes_decipher(u8 *key, u8 hdrlen,
+			       u8 *pframe, u32 plen)
 {
 	static u8 message[MAX_MSG_SIZE];
 	uint qc_exists, a4_exists, i, j, payload_remainder,
-- 
2.55.0

[PATCH v3 2/6] staging: rtl8723bs: use bool for flags in rtw_security.c

[Pranav Desai]

Replace 'uint' with 'bool' for the 'qc_exists' and 'a4_exists' variables
since they are only used to track true/false state.

Update their assignments from 1/0 to true/false to match the standard
boolean type.

Signed-off-by: Pranav Desai <contact.pranavdesai@gmail.com>
---
 drivers/staging/rtl8723bs/core/rtw_security.c | 34 ++++++++++---------
 1 file changed, 18 insertions(+), 16 deletions(-)

diff --git a/drivers/staging/rtl8723bs/core/rtw_security.c b/drivers/staging/rtl8723bs/core/rtw_security.c
index af461971c827..34e35c0eccbc 100644
--- a/drivers/staging/rtl8723bs/core/rtw_security.c
+++ b/drivers/staging/rtl8723bs/core/rtw_security.c
@@ -628,8 +628,8 @@ static void aes128k128d(u8 *key, u8 *data, u8 *ciphertext)
 /* nonce                                        */
 /************************************************/
 static void construct_mic_iv(u8 *mic_iv,
-			     signed int qc_exists,
-			     signed int a4_exists,
+			     bool qc_exists,
+			     bool a4_exists,
 			     u8 *mpdu,
 			     uint payload_length,
 			     u8 *pn_vector,
@@ -702,8 +702,8 @@ static void construct_mic_header1(u8 *mic_header1,
 /************************************************/
 static void construct_mic_header2(u8 *mic_header2,
 				  u8 *mpdu,
-				  signed int a4_exists,
-				  signed int qc_exists)
+				  bool a4_exists,
+				  bool qc_exists)
 {
 		signed int i;
 
@@ -783,7 +783,8 @@ static void construct_ctr_preload(u8 *ctr_preload,
 static signed int aes_cipher(u8 *key, u16 hdrlen,
 			     u8 *pframe, u32 plen)
 {
-	uint	qc_exists, a4_exists, i, j, payload_remainder,
+	bool qc_exists, a4_exists;
+	uint	i, j, payload_remainder,
 		num_blocks, payload_index;
 
 	u8 pn_vector[6];
@@ -803,14 +804,14 @@ static signed int aes_cipher(u8 *key, u16 hdrlen,
 	frsubtype = frsubtype >> 4;
 
 	if ((hdrlen == WLAN_HDR_A3_LEN) || (hdrlen ==  WLAN_HDR_A3_QOS_LEN))
-		a4_exists = 0;
+		a4_exists = false;
 	else
-		a4_exists = 1;
+		a4_exists = true;
 
 	if (((frtype | frsubtype) == WIFI_DATA_CFACK) ||
 	    ((frtype | frsubtype) == WIFI_DATA_CFPOLL) ||
 	    ((frtype | frsubtype) == WIFI_DATA_CFACKPOLL)) {
-		qc_exists = 1;
+		qc_exists = true;
 		if (hdrlen !=  WLAN_HDR_A3_QOS_LEN)
 			hdrlen += 2;
 
@@ -822,9 +823,9 @@ static signed int aes_cipher(u8 *key, u16 hdrlen,
 		if (hdrlen !=  WLAN_HDR_A3_QOS_LEN)
 			hdrlen += 2;
 
-		qc_exists = 1;
+		qc_exists = true;
 	} else {
-		qc_exists = 0;
+		qc_exists = false;
 	}
 
 	pn_vector[0] = pframe[hdrlen];
@@ -984,7 +985,8 @@ static signed int aes_decipher(u8 *key, u8 hdrlen,
 			       u8 *pframe, u32 plen)
 {
 	static u8 message[MAX_MSG_SIZE];
-	uint qc_exists, a4_exists, i, j, payload_remainder,
+	bool qc_exists, a4_exists;
+	uint i, j, payload_remainder,
 			num_blocks, payload_index;
 	signed int res = _SUCCESS;
 	u8 pn_vector[6];
@@ -1018,14 +1020,14 @@ static signed int aes_decipher(u8 *key, u8 hdrlen,
 	pn_vector[5]  = pframe[hdrlen + 7];
 
 	if ((hdrlen == WLAN_HDR_A3_LEN) || (hdrlen ==  WLAN_HDR_A3_QOS_LEN))
-		a4_exists = 0;
+		a4_exists = false;
 	else
-		a4_exists = 1;
+		a4_exists = true;
 
 	if (((frtype | frsubtype) == WIFI_DATA_CFACK) ||
 	    ((frtype | frsubtype) == WIFI_DATA_CFPOLL) ||
 	    ((frtype | frsubtype) == WIFI_DATA_CFACKPOLL)) {
-		qc_exists = 1;
+		qc_exists = true;
 		if (hdrlen !=  WLAN_HDR_A3_QOS_LEN)
 			hdrlen += 2;
 
@@ -1037,9 +1039,9 @@ static signed int aes_decipher(u8 *key, u8 hdrlen,
 		if (hdrlen !=  WLAN_HDR_A3_QOS_LEN)
 			hdrlen += 2;
 
-		qc_exists = 1;
+		qc_exists = true;
 	} else {
-		qc_exists = 0;
+		qc_exists = false;
 	}
 
 	/*  now, decrypt pframe with hdrlen offset and plen long */
-- 
2.55.0

[PATCH v3 3/6] staging: rtl8723bs: use u8 for frame types in rtw_security.c

[Pranav Desai]

Replace 'uint' with 'u8' for the 'frtype' and 'frsubtype' variables
in rtw_security.c.

These variables are populated by the GetFrameType() and GetFrameSubType()
macros, which read a 16-bit value but apply a bitwise mask (bits 2-3 for
type, bits 2-7 for subtype). Because the highest possible value after
masking is 252, these variables are guaranteed to fit within an 8-bit
unsigned integer.

Signed-off-by: Pranav Desai <contact.pranavdesai@gmail.com>
---
 drivers/staging/rtl8723bs/core/rtw_security.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/drivers/staging/rtl8723bs/core/rtw_security.c b/drivers/staging/rtl8723bs/core/rtw_security.c
index 34e35c0eccbc..676587fabfe4 100644
--- a/drivers/staging/rtl8723bs/core/rtw_security.c
+++ b/drivers/staging/rtl8723bs/core/rtw_security.c
@@ -633,7 +633,7 @@ static void construct_mic_iv(u8 *mic_iv,
 			     u8 *mpdu,
 			     uint payload_length,
 			     u8 *pn_vector,
-			     uint frtype)
+			     u8 frtype)
 {
 		signed int i;
 
@@ -669,7 +669,7 @@ static void construct_mic_iv(u8 *mic_iv,
 static void construct_mic_header1(u8 *mic_header1,
 				  signed int header_length,
 				  u8 *mpdu,
-				  uint frtype)
+				  u8 frtype)
 {
 		mic_header1[0] = (u8)((header_length - 2) / 256);
 		mic_header1[1] = (u8)((header_length - 2) % 256);
@@ -752,7 +752,7 @@ static void construct_ctr_preload(u8 *ctr_preload,
 				  u8 *mpdu,
 				  u8 *pn_vector,
 				  signed int c,
-				  uint frtype)
+				  u8 frtype)
 {
 	signed int i = 0;
 
@@ -798,8 +798,8 @@ static signed int aes_cipher(u8 *key, u16 hdrlen,
 	u8 aes_out[16] = {};
 	u8 padded_buffer[16] = {};
 	u8 mic[8];
-	uint	frtype  = GetFrameType(pframe);
-	uint	frsubtype  = GetFrameSubType(pframe);
+	u8 frtype  = GetFrameType(pframe);
+	u8 frsubtype  = GetFrameSubType(pframe);
 
 	frsubtype = frsubtype >> 4;
 
@@ -1001,8 +1001,8 @@ static signed int aes_decipher(u8 *key, u8 hdrlen,
 	u8 padded_buffer[16] = {};
 	u8 mic[8];
 
-	uint frtype  = GetFrameType(pframe);
-	uint frsubtype  = GetFrameSubType(pframe);
+	u8 frtype  = GetFrameType(pframe);
+	u8 frsubtype  = GetFrameSubType(pframe);
 
 	frsubtype = frsubtype >> 4;
 
-- 
2.55.0

[PATCH v3 4/6] staging: rtl8723bs: use explicit types for math variables and iterators

[Pranav Desai]

Replace 'uint' with explicit types for variables used in math operations
and loops inside the aes_cipher() and aes_decipher() functions:

- Change 'payload_remainder' to u8 because it is calculated as (plen % 16),
which guarantees a range of 0-15.
- Change 'payload_index' to u32 because it indexes into the payload, which
is bounded by plen (u32).
- Change 'num_blocks' to u32 since it is derived from 'plen' (u32).
- Change loop iterators (i, j) to standard 'unsigned int'.

Signed-off-by: Pranav Desai <contact.pranavdesai@gmail.com>
---
 drivers/staging/rtl8723bs/core/rtw_security.c | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/drivers/staging/rtl8723bs/core/rtw_security.c b/drivers/staging/rtl8723bs/core/rtw_security.c
index 676587fabfe4..ff8af1a29b22 100644
--- a/drivers/staging/rtl8723bs/core/rtw_security.c
+++ b/drivers/staging/rtl8723bs/core/rtw_security.c
@@ -784,8 +784,10 @@ static signed int aes_cipher(u8 *key, u16 hdrlen,
 			     u8 *pframe, u32 plen)
 {
 	bool qc_exists, a4_exists;
-	uint	i, j, payload_remainder,
-		num_blocks, payload_index;
+	u8 payload_remainder;
+	u32 payload_index;
+	u32 num_blocks;
+	unsigned int i, j;
 
 	u8 pn_vector[6];
 	u8 mic_iv[16] = {};
@@ -986,8 +988,10 @@ static signed int aes_decipher(u8 *key, u8 hdrlen,
 {
 	static u8 message[MAX_MSG_SIZE];
 	bool qc_exists, a4_exists;
-	uint i, j, payload_remainder,
-			num_blocks, payload_index;
+	u8 payload_remainder;
+	u32 payload_index;
+	u32 num_blocks;
+	unsigned int i, j;
 	signed int res = _SUCCESS;
 	u8 pn_vector[6];
 	u8 mic_iv[16] = {};
-- 
2.55.0

[PATCH v3 5/6] staging: rtl8723bs: use u32 for payload_length in rtw_security.c

[Pranav Desai]

Replace 'uint' with 'u32' for the 'payload_length' parameter in the
construct_mic_iv() helper function.

This parameter is passed directly from the 'plen' variable in the caller
functions, which has been explicitly typed as u32.

Signed-off-by: Pranav Desai <contact.pranavdesai@gmail.com>
---
 drivers/staging/rtl8723bs/core/rtw_security.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/staging/rtl8723bs/core/rtw_security.c b/drivers/staging/rtl8723bs/core/rtw_security.c
index ff8af1a29b22..8b67985f6e06 100644
--- a/drivers/staging/rtl8723bs/core/rtw_security.c
+++ b/drivers/staging/rtl8723bs/core/rtw_security.c
@@ -631,7 +631,7 @@ static void construct_mic_iv(u8 *mic_iv,
 			     bool qc_exists,
 			     bool a4_exists,
 			     u8 *mpdu,
-			     uint payload_length,
+			     u32 payload_length,
 			     u8 *pn_vector,
 			     u8 frtype)
 {
-- 
2.55.0

[PATCH v3 6/6] staging: rtl8723bs: use u32 for length variables in rtw_BIP_verify()

[Pranav Desai]

Replace 'uint' with 'u32' for the 'len' and 'ori_len' variables in
the rtw_BIP_verify() function.

These variables are calculated directly from 'pattrib->pkt_len',
which is defined as a u32 in struct pkt_attrib. Explicitly typing
them as u32 prevents potential bounds or overflow issues during
subsequent length calculations.

Signed-off-by: Pranav Desai <contact.pranavdesai@gmail.com>
---
 drivers/staging/rtl8723bs/core/rtw_security.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/staging/rtl8723bs/core/rtw_security.c b/drivers/staging/rtl8723bs/core/rtw_security.c
index 8b67985f6e06..679c957c9c63 100644
--- a/drivers/staging/rtl8723bs/core/rtw_security.c
+++ b/drivers/staging/rtl8723bs/core/rtw_security.c
@@ -1272,7 +1272,7 @@ u32 rtw_BIP_verify(struct adapter *padapter, u8 *precvframe)
 	u8 *pframe;
 	u8 *BIP_AAD, *p;
 	u32 res = _FAIL;
-	uint len, ori_len;
+	u32 len, ori_len;
 	struct ieee80211_hdr *pwlanhdr;
 	u8 mic[16];
 	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-- 
2.55.0