c++: Add C++20 #__VA_OPT__ support

The following patch implements C++20 # __VA_OPT__ (...) support.

Testcases cover what I came up with myself and what LLVM has for #__VA_OPT__

in its testsuite and the string literals are identical between the two

compilers on the va-opt-5.c testcase.

2021-08-17  Jakub Jelinek  <jakub@redhat.com>

libcpp/

	* macro.c (vaopt_state): Add m_stringify member.

	(vaopt_state::vaopt_state): Initialize it.

	(vaopt_state::update): Overwrite it.

	(vaopt_state::stringify): New method.

	(stringify_arg): Replace arg argument with first, count arguments

	and add va_opt argument.  Use first instead of arg->first and

	count instead of arg->count, for va_opt add paste_tokens handling.

	(paste_tokens): Fix up len calculation.  Don't spell rhs twice,

	instead use %.*s to supply lhs and rhs spelling lengths.  Don't call

	_cpp_backup_tokens here.

	(paste_all_tokens): Call it here instead.

	(replace_args): Adjust stringify_arg caller.  For vaopt_state::END

	if stringify is true handle __VA_OPT__ stringification.

	(create_iso_definition): Handle # __VA_OPT__ similarly to # macro_arg.

gcc/testsuite/

	* c-c++-common/cpp/va-opt-5.c: New test.

	* c-c++-common/cpp/va-opt-6.c: New test.

--- libcpp/macro.c

+++ libcpp/macro.c

@@ -118,6 +118,7 @@ class vaopt_state {

     m_arg (arg),

     m_variadic (is_variadic),

     m_last_was_paste (false),

+    m_stringify (false),

     m_state (0),

     m_paste_location (0),

     m_location (0),

@@ -145,6 +146,7 @@ class vaopt_state {

 	  }

 	++m_state;

 	m_location = token->src_loc;

+	m_stringify = (token->flags & STRINGIFY_ARG) != 0;

 	return BEGIN;

       }

     else if (m_state == 1)

@@ -234,6 +236,12 @@ class vaopt_state {

     return m_state == 0;

   }

+  /* Return true for # __VA_OPT__.  */

+  bool stringify () const

+  {

+    return m_stringify;

+  }

+

  private:

   /* The cpp_reader.  */

@@ -247,6 +255,8 @@ class vaopt_state {

   /* If true, the previous token was ##.  This is used to detect when

      a paste occurs at the end of the sequence.  */

   bool m_last_was_paste;

+  /* True for #__VA_OPT__.  */

+  bool m_stringify;

   /* The state variable:

      0 means not parsing

@@ -284,7 +294,8 @@ static _cpp_buff *collect_args (cpp_read

 static cpp_context *next_context (cpp_reader *);

 static const cpp_token *padding_token (cpp_reader *, const cpp_token *);

 static const cpp_token *new_string_token (cpp_reader *, uchar *, unsigned int);

-static const cpp_token *stringify_arg (cpp_reader *, macro_arg *);

+static const cpp_token *stringify_arg (cpp_reader *, const cpp_token **,

+				       unsigned int, bool);

 static void paste_all_tokens (cpp_reader *, const cpp_token *);

 static bool paste_tokens (cpp_reader *, location_t,

 			  const cpp_token **, const cpp_token *);

@@ -812,10 +823,11 @@ cpp_quote_string (uchar *dest, const uch

   return dest;

 }

-/* Convert a token sequence ARG to a single string token according to

-   the rules of the ISO C #-operator.  */

+/* Convert a token sequence FIRST to FIRST+COUNT-1 to a single string token

+   according to the rules of the ISO C #-operator.  */

 static const cpp_token *

-stringify_arg (cpp_reader *pfile, macro_arg *arg)

+stringify_arg (cpp_reader *pfile, const cpp_token **first, unsigned int count,

+	       bool va_opt)

 {

   unsigned char *dest;

   unsigned int i, escape_it, backslash_count = 0;

@@ -828,9 +840,27 @@ stringify_arg (cpp_reader *pfile, macro_

   *dest++ = '"';

   /* Loop, reading in the argument's tokens.  */

-  for (i = 0; i < arg->count; i++)

+  for (i = 0; i < count; i++)

     {

-      const cpp_token *token = arg->first[i];

+      const cpp_token *token = first[i];

+

+      if (va_opt && (token->flags & PASTE_LEFT))

+	{

+	  location_t virt_loc = pfile->invocation_location;

+	  const cpp_token *rhs;

+	  do

+	    {

+	      if (i == count)

+		abort ();

+	      rhs = first[++i];

+	      if (!paste_tokens (pfile, virt_loc, &token, rhs))

+		{

+		  --i;

+		  break;

+		}

+	    }

+	  while (rhs->flags & PASTE_LEFT);

+	}

       if (token->type == CPP_PADDING)

 	{

@@ -917,7 +947,7 @@ paste_tokens (cpp_reader *pfile, locatio

   cpp_token *lhs;

   unsigned int len;

-  len = cpp_token_len (*plhs) + cpp_token_len (rhs) + 1;

+  len = cpp_token_len (*plhs) + cpp_token_len (rhs) + 2;

   buf = (unsigned char *) alloca (len);

   end = lhsend = cpp_spell_token (pfile, *plhs, buf, true);

@@ -943,8 +973,10 @@ paste_tokens (cpp_reader *pfile, locatio

       location_t saved_loc = lhs->src_loc;

       _cpp_pop_buffer (pfile);

-      _cpp_backup_tokens (pfile, 1);

-      *lhsend = '\0';

+

+      unsigned char *rhsstart = lhsend;

+      if ((*plhs)->type == CPP_DIV && rhs->type != CPP_EQ)

+	rhsstart++;

       /* We have to remove the PASTE_LEFT flag from the old lhs, but

 	 we want to keep the new location.  */

@@ -956,8 +988,10 @@ paste_tokens (cpp_reader *pfile, locatio

       /* Mandatory error for all apart from assembler.  */

       if (CPP_OPTION (pfile, lang) != CLK_ASM)

 	cpp_error_with_line (pfile, CPP_DL_ERROR, location, 0,

-	 "pasting \"%s\" and \"%s\" does not give a valid preprocessing token",

-		   buf, cpp_token_as_text (pfile, rhs));

+			     "pasting \"%.*s\" and \"%.*s\" does not give "

+			     "a valid preprocessing token",

+			     (int) (lhsend - buf), buf,

+			     (int) (end - rhsstart), rhsstart);

       return false;

     }

@@ -1033,7 +1067,10 @@ paste_all_tokens (cpp_reader *pfile, con

 	    abort ();

 	}

       if (!paste_tokens (pfile, virt_loc, &lhs, rhs))

-	break;

+	{

+	  _cpp_backup_tokens (pfile, 1);

+	  break;

+	}

     }

   while (rhs->flags & PASTE_LEFT);

@@ -1900,7 +1937,8 @@ replace_args (cpp_reader *pfile, cpp_has

 	if (src->flags & STRINGIFY_ARG)

 	  {

 	    if (!arg->stringified)

-	      arg->stringified = stringify_arg (pfile, arg);

+	      arg->stringified = stringify_arg (pfile, arg->first, arg->count,

+						false);

 	  }

 	else if ((src->flags & PASTE_LEFT)

 		 || (src != macro->exp.tokens && (src[-1].flags & PASTE_LEFT)))

@@ -2023,6 +2061,24 @@ replace_args (cpp_reader *pfile, cpp_has

 		  paste_flag = tokens_buff_last_token_ptr (buff);

 		}

+	      if (vaopt_tracker.stringify ())

+		{

+		  unsigned int count

+		    = start ? paste_flag - start : tokens_buff_count (buff);

+		  const cpp_token *t

+		    = stringify_arg (pfile,

+				     start ? start + 1

+				     : (const cpp_token **) (buff->base),

+				     count, true);

+		  while (count--)

+		    tokens_buff_remove_last_token (buff);

+		  if (src->flags & PASTE_LEFT)

+		    copy_paste_flag (pfile, &t, src);

+		  tokens_buff_add_token (buff, virt_locs,

+					 t, t->src_loc, t->src_loc,

+					 NULL, 0);

+		  continue;

+		}

 	      if (start && paste_flag == start && (*start)->flags & PASTE_LEFT)

 		/* If __VA_OPT__ expands to nothing (either because __VA_ARGS__

 		   is empty or because it is __VA_OPT__() ), drop PASTE_LEFT

@@ -3584,7 +3640,10 @@ create_iso_definition (cpp_reader *pfile

 	 function-like macros when lexing the subsequent token.  */

       if (macro->count > 1 && token[-1].type == CPP_HASH && macro->fun_like)

 	{

-	  if (token->type == CPP_MACRO_ARG)

+	  if (token->type == CPP_MACRO_ARG

+	      || (macro->variadic

+		  && token->type == CPP_NAME

+		  && token->val.node.node == pfile->spec_nodes.n__VA_OPT__))

 	    {

 	      if (token->flags & PREV_WHITE)

 		token->flags |= SP_PREV_WHITE;

--- gcc/testsuite/c-c++-common/cpp/va-opt-5.c

+++ gcc/testsuite/c-c++-common/cpp/va-opt-5.c

@@ -0,0 +1,67 @@

+/* { dg-do run } */

+/* { dg-options "-std=gnu99" { target c } } */

+/* { dg-options "-std=c++20" { target c++ } } */

+

+#define lparen (

+#define a0 fooa0

+#define a1  fooa1 a0

+#define a2  fooa2 a1

+#define a3  fooa3 a2

+#define a() b lparen )

+#define b() c lparen )

+#define c() d lparen )

+#define g h

+#define i(j) j

+#define f(...) #__VA_OPT__(g i(0))

+#define k(x,...) # __VA_OPT__(x) #x #__VA_OPT__(__VA_ARGS__)

+#define l(x,...) #__VA_OPT__(a1 x)

+#define m(x,...) "a()" #__VA_OPT__(a3 __VA_ARGS__ x ## __VA_ARGS__ ## x ## c a3) "a()"

+#define n(x,...) = #__VA_OPT__(a3 __VA_ARGS__ x ## __VA_ARGS__ ## x ## c a3) #x #__VA_OPT__(a0 __VA_ARGS__ x ## __VA_ARGS__ ## x ## c a0) ;

+#define o(x, ...) #__VA_OPT__(x##x x##x)

+#define p(x, ...) #__VA_OPT__(_Pragma ("foobar"))

+#define q(...) #__VA_OPT__(/* foo */x/* bar */)

+const char *v1 = f();

+const char *v2 = f(123);

+const char *v3 = k(1);

+const char *v4 = k(1, 2, 3 );

+const char *v5 = l(a());

+const char *v6 = l(a1 a(), 1);

+const char *v7 = m();

+const char *v8 = m(,);

+const char *v9 = m(,a3);

+const char *v10 = m(a3,a(),a0);

+const char *v11 n()

+const char *v12 n(,)

+const char *v13 n(,a0)

+const char *v14 n(a0, a(),a0)

+const char *v15 = o(, 0);

+const char *v16 = p(0);

+const char *v17 = p(0, 1);

+const char *v18 = q();

+const char *v19 = q(1);

+

+int

+main ()

+{

+  if (__builtin_strcmp (v1, "")

+      || __builtin_strcmp (v2, "g i(0)")

+      || __builtin_strcmp (v3, "1")

+      || __builtin_strcmp (v4, "112, 3")

+      || __builtin_strcmp (v5, "")

+      || __builtin_strcmp (v6, "a1 fooa1 fooa0 b ( )")

+      || __builtin_strcmp (v7, "a()a()")

+      || __builtin_strcmp (v8, "a()a()")

+      || __builtin_strcmp (v9, "a()a3 fooa3 fooa2 fooa1 fooa0 a3c a3a()")

+      || __builtin_strcmp (v10, "a()a3 b ( ),fooa0 a3a(),a0a3c a3a()")

+      || __builtin_strcmp (v11, "")

+      || __builtin_strcmp (v12, "")

+      || __builtin_strcmp (v13, "a3 fooa0 a0c a3a0 fooa0 a0c a0")

+      || __builtin_strcmp (v14, "a3 b ( ),fooa0 a0a(),a0a0c a3a0a0 b ( ),fooa0 a0a(),a0a0c a0")

+      || __builtin_strcmp (v15, "")

+      || __builtin_strcmp (v16, "")

+      || __builtin_strcmp (v17, "_Pragma (\"foobar\")")

+      || __builtin_strcmp (v18, "")

+      || __builtin_strcmp (v19, "x"))

+    __builtin_abort ();

+  return 0;

+}

--- gcc/testsuite/c-c++-common/cpp/va-opt-6.c

+++ gcc/testsuite/c-c++-common/cpp/va-opt-6.c

@@ -0,0 +1,17 @@

+/* { dg-do preprocess } */

+/* { dg-options "-std=gnu99" { target c } } */

+/* { dg-options "-std=c++20" { target c++ } } */

+

+#define a ""

+#define b(...) a ## #__VA_OPT__(1)	/* { dg-error "pasting \"a\" and \"\"\"\" does not give a valid preprocessing token" } */

+#define c(...) a ## #__VA_OPT__(1)	/* { dg-error "pasting \"a\" and \"\"1\"\" does not give a valid preprocessing token" } */

+#define d(...) #__VA_OPT__(1) ## !

+#define e(...) #__VA_OPT__(1) ## !

+#define f(...) #__VA_OPT__(. ## !)

+#define g(...) #__VA_OPT__(. ## !)

+b()

+c(1)

+d(   )		/* { dg-error "pasting \"\"\"\" and \"!\" does not give a valid preprocessing token" } */

+e(  1 )		/* { dg-error "pasting \"\"1\"\" and \"!\" does not give a valid preprocessing token" } */

+f()

+g(0)		/* { dg-error "pasting \".\" and \"!\" does not give a valid preprocessing token" } */