diff --git a/contrib/pg_standby/pg_standby.c b/contrib/pg_standby/pg_standby.c index ee1fbd7..946239c 100644 --- a/contrib/pg_standby/pg_standby.c +++ b/contrib/pg_standby/pg_standby.c @@ -611,7 +611,7 @@ usage(void) printf(" -w MAXWAITTIME max seconds to wait for a file (0=no limit) (default=0)\n"); printf(" -?, --help show this help, then exit\n"); printf("\n" - "Main intended use as restore_command in recovery.conf:\n" + "Main intended use as restore_command in postgresql.conf:\n" " restore_command = 'pg_standby [OPTION]... ARCHIVELOCATION %%f %%p %%r'\n" "e.g.\n" " restore_command = 'pg_standby /mnt/server/archiverdir %%f %%p %%r'\n"); diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml index 3fa5efd..780f40d 100644 --- a/doc/src/sgml/backup.sgml +++ b/doc/src/sgml/backup.sgml @@ -1220,7 +1220,7 @@ SELECT pg_stop_backup(); - Create a recovery command file recovery.conf in the cluster + Create a file recovery.signal in the cluster data directory (see ). You might also want to temporarily modify pg_hba.conf to prevent ordinary users from connecting until you are sure the recovery was successful. @@ -1232,10 +1232,9 @@ SELECT pg_stop_backup(); proceed to read through the archived WAL files it needs. Should the recovery be terminated because of an external error, the server can simply be restarted and it will continue recovery. Upon completion - of the recovery process, the server will rename - recovery.conf to recovery.done (to prevent - accidentally re-entering recovery mode later) and then - commence normal database operations. + of the recovery process, the server will remove + recovery.signal (to prevent accidentally re-entering + recovery mode later) and then commence normal database operations. @@ -1249,12 +1248,8 @@ SELECT pg_stop_backup(); - The key part of all this is to set up a recovery configuration file that - describes how you want to recover and how far the recovery should - run. You can use recovery.conf.sample (normally - located in the installation's share/ directory) as a - prototype. The one thing that you absolutely must specify in - recovery.conf is the restore_command, + The key part of all this is to set up a recovery configuration. + The one thing that you absolutely must specify is the restore_command, which tells PostgreSQL how to retrieve archived WAL file segments. Like the archive_command, this is a shell command string. It can contain %f, which is @@ -1316,7 +1311,7 @@ restore_command = 'cp /mnt/server/archivedir/%f %p' If you want to recover to some previous point in time (say, right before the junior DBA dropped your main transaction table), just specify the - required stopping point in recovery.conf. You can specify + required stopping point. You can specify the stop point, known as the recovery target, either by date/time, named restore point or by completion of a specific transaction ID. As of this writing only the date/time and named restore point options @@ -1414,8 +1409,8 @@ restore_command = 'cp /mnt/server/archivedir/%f %p' that was current when the base backup was taken. If you wish to recover into some child timeline (that is, you want to return to some state that was itself generated after a recovery attempt), you need to specify the - target timeline ID in recovery.conf. You cannot recover into - timelines that branched off earlier than the base backup. + target timeline ID in . You + cannot recover into timelines that branched off earlier than the base backup. diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index f11b8f7..7e81a88 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3202,11 +3202,11 @@ include_dir 'conf.d' application_name setting of the standby, as set in the standby's connection information. In case of a physical replication standby, this should be set in the primary_conninfo - setting in recovery.conf; the default - is walreceiver. For logical replication, this can - be set in the connection information of the subscription, and it - defaults to the subscription name. For other replication stream - consumers, consult their documentation. + setting; the default is walreceiver. + For logical replication, this can be set in the connection + information of the subscription, and it defaults to the + subscription name. For other replication stream consumers, + consult their documentation. This parameter specifies a list of standby servers using diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 9a7f683..725e0d6 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -19057,7 +19057,7 @@ postgres=# select pg_start_backup('label_goes_here'); pg_create_restore_point creates a named write-ahead log record that can be used as recovery target, and returns the corresponding write-ahead log location. The given name can then be used with - to specify the point up to which + to specify the point up to which recovery will proceed. Avoid creating multiple restore points with the same name, since recovery will stop at the first one whose name matches the recovery target. diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml index 6f57362..77fe990 100644 --- a/doc/src/sgml/high-availability.sgml +++ b/doc/src/sgml/high-availability.sgml @@ -686,10 +686,9 @@ protocol to make nodes agree on a serializable transactional order. To set up the standby server, restore the base backup taken from primary - server (see ). Create a recovery - command file recovery.conf in the standby's cluster data - directory, and turn on standby_mode. Set - restore_command to a simple command to copy files from + server (see ). Create a file + standby.signal in the standby's cluster data + directory. Set to a simple command to copy files from the WAL archive. If you plan to have multiple standby servers for high availability purposes, set recovery_target_timeline to latest, to make the standby server follow the timeline change @@ -699,7 +698,7 @@ protocol to make nodes agree on a serializable transactional order. Do not use pg_standby or similar tools with the built-in standby mode - described here. restore_command should return immediately + described here. should return immediately if the file does not exist; the server will retry the command again if necessary. See for using tools like pg_standby. @@ -708,11 +707,11 @@ protocol to make nodes agree on a serializable transactional order. If you want to use streaming replication, fill in - primary_conninfo with a libpq connection string, including + with a libpq connection string, including the host name (or IP address) and any additional details needed to connect to the primary server. If the primary needs a password for authentication, the password needs to be specified in - primary_conninfo as well. + as well. @@ -735,9 +734,8 @@ protocol to make nodes agree on a serializable transactional order. - A simple example of a recovery.conf is: + A simple example of configuration is: -standby_mode = 'on' primary_conninfo = 'host=192.168.1.50 port=5432 user=foo password=foopass options=''-c wal_sender_timeout=5000''' restore_command = 'cp /path/to/archive/%f %p' archive_cleanup_command = 'pg_archivecleanup /path/to/archive %r' @@ -793,8 +791,8 @@ archive_cleanup_command = 'pg_archivecleanup /path/to/archive %r' To use streaming replication, set up a file-based log-shipping standby server as described in . The step that turns a file-based log-shipping standby into streaming replication - standby is setting primary_conninfo setting in the - recovery.conf file to point to the primary server. Set + standby is setting primary_conninfo setting + to point to the primary server. Set and authentication options (see pg_hba.conf) on the primary so that the standby server can connect to the replication pseudo-database on the primary @@ -854,14 +852,14 @@ host replication foo 192.168.1.100/32 md5 The host name and port number of the primary, connection user name, - and password are specified in the recovery.conf file. + and password are specified in the . The password can also be set in the ~/.pgpass file on the standby (specify replication in the database field). For example, if the primary is running on host IP 192.168.1.50, port 5432, the account name for replication is foo, and the password is foopass, the administrator - can add the following line to the recovery.conf file on the + can add the following line to the postgresql.conf file on the standby: @@ -967,16 +965,14 @@ postgres=# SELECT * FROM pg_create_physical_replication_slot('node_a_slot'); node_a_slot | postgres=# SELECT slot_name, slot_type, active FROM pg_replication_slots; - slot_name | slot_type | active + slot_name | slot_type | active -------------+-----------+-------- node_a_slot | physical | f (1 row) To configure the standby to use this slot, primary_slot_name - should be configured in the standby's recovery.conf. - Here is a simple example: + should be configured on the standby. Here is a simple example: -standby_mode = 'on' primary_conninfo = 'host=192.168.1.50 port=5432 user=foo password=foopass' primary_slot_name = 'node_a_slot' @@ -1473,9 +1469,9 @@ synchronous_standby_names = 'ANY 2 (s1, s2, s3)' To trigger failover of a log-shipping standby server, run pg_ctl promote or create a trigger - file with the file name and path specified by the trigger_file - setting in recovery.conf. If you're planning to use - pg_ctl promote to fail over, trigger_file is + file with the file name and path specified by the promote_signal_file + setting. If you're planning to use + pg_ctl promote to fail over, promote_signal_file is not required. If you're setting up the reporting servers that are only used to offload read-only queries from the primary, not for high availability purposes, you don't need to promote it. @@ -1488,11 +1484,8 @@ synchronous_standby_names = 'ANY 2 (s1, s2, s3)' An alternative to the built-in standby mode described in the previous sections is to use a restore_command that polls the archive location. - This was the only option available in versions 8.4 and below. In this - setup, set standby_mode off, because you are implementing - the polling required for standby operation yourself. See the - module for a reference - implementation of this. + This was the only option available in versions 8.4 and below. See the + module for a reference implementation of this. @@ -1519,14 +1512,13 @@ synchronous_standby_names = 'ANY 2 (s1, s2, s3)' The magic that makes the two loosely coupled servers work together is simply a restore_command used on the standby that, when asked for the next WAL file, waits for it to become available from - the primary. The restore_command is specified in the - recovery.conf file on the standby server. Normal recovery - processing would request a file from the WAL archive, reporting failure - if the file was unavailable. For standby processing it is normal for - the next WAL file to be unavailable, so the standby must wait for - it to appear. For files ending in - .history there is no need to wait, and a non-zero return - code must be returned. A waiting restore_command can be + the primary. Normal recovery processing would request a file from the WAL + archive, reporting failure if the file was unavailable. For standby + processing it is normal for the next WAL file to be unavailable, + so the standby must wait for it to appear. For files ending in + .backup or .history + there is no need to wait, and a non-zero return code must be returned. + A waiting restore_command can be written as a custom script that loops after polling for the existence of the next WAL file. There must also be some way to trigger failover, which should interrupt the restore_command, break the loop and @@ -1608,9 +1600,8 @@ if (!triggered) Begin recovery on the standby server from the local WAL - archive, using a recovery.conf that specifies a - restore_command that waits as described - previously (see ). + archive, using restore_command that waits + as described previously (see ). @@ -2105,7 +2096,7 @@ if (!triggered) If hot_standby is on in postgresql.conf - (the default value) and there is a recovery.conf + (the default value) and there is a standby.signal file present, the server will run in Hot Standby mode. However, it may take some time for Hot Standby connections to be allowed, because the server will not accept connections until it has completed diff --git a/doc/src/sgml/pgstandby.sgml b/doc/src/sgml/pgstandby.sgml index 2cc58fe..d8aded4 100644 --- a/doc/src/sgml/pgstandby.sgml +++ b/doc/src/sgml/pgstandby.sgml @@ -47,7 +47,7 @@ To configure a standby server to use pg_standby, put this into its - recovery.conf configuration file: + postgresql.conf configuration file: restore_command = 'pg_standby archiveDir %f %p %r' diff --git a/doc/src/sgml/recovery-config.sgml b/doc/src/sgml/recovery-config.sgml index 92825fd..5cdca61 100644 --- a/doc/src/sgml/recovery-config.sgml +++ b/doc/src/sgml/recovery-config.sgml @@ -11,23 +11,44 @@ This chapter describes the settings available in the - recovery.confrecovery.conf - file. They apply only for the duration of the + postgresql.conf + file that apply only for the duration of the recovery. They must be reset for any subsequent recovery you wish to perform. They cannot be changed once recovery has begun. - Settings in recovery.conf are specified in the format - name = 'value'. One parameter is specified per line. - Hash marks (#) designate the rest of the - line as a comment. To embed a single quote in a parameter - value, write two quotes (''). + The database server can also be started in recovery, a term that covers + using the server as a standby or for executing a targeted recovery. Typically + standby mode would be used to provide high availability and/or read + scalability, whereas a targeted recovery is used to recover from data loss. - A sample file, share/recovery.conf.sample, - is provided in the installation's share/ directory. + To start the server in standby mode create file + called standby.signalstandby.signal + in the data directory. The server will enter recovery and + will not stop recovery when the end of archived WAL is reached, but + will keep trying to continue recovery by connecting to the sending server as + specified by the primary_conninfo setting and/or by + fetching new WAL segments using restore_command + In this mode you may use parameters + in both and + sections. Parameters from + will not be used. + + + + To start the server in targeted recovery create a file called + recovery.signalrecovery.signal + in the data directory. + If both standby.signal and recovery.signal files are + created, standby mode takes precedence. Targeted recovery mode will end when + end of archived WAL is reached, or when recovery_target is reached. + In this mode you may use parameters from both + and + sections. Parameters from + will not be used. @@ -45,7 +66,7 @@ The local shell command to execute to retrieve an archived segment of the WAL file series. This parameter is required for archive recovery, - but optional for streaming replication. + but optional for standby mode. Any %f in the string is replaced by the name of the file to retrieve from the archive, and any %p is replaced by the copy destination path name @@ -154,99 +175,101 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows Recovery Target Settings - By default, recovery will recover to the end of the WAL log. The - following parameters can be used to specify an earlier stopping point. - At most one of recovery_target, - recovery_target_lsn, recovery_target_name, - recovery_target_time, or recovery_target_xid - can be used; if more than one of these is specified in the configuration - file, the last entry will be used. + By default, recovery will process to the end of the WAL log. An earlier + stopping point may be specified using recovery_target_type + and in most cases also recovery_target_value, plus the optional + parameters recovery_target_inclusive, + recovery_target_timeline and recovery_target_action. - - recovery_target = 'immediate' + + recovery_target_type (enum) - recovery_target recovery parameter + recovery_target_type targeted recovery parameter - This parameter specifies that recovery should end as soon as a - consistent state is reached, i.e. as early as possible. When restoring - from an online backup, this means the point where taking the backup - ended. + recovery_target_type specifies the search criteria used for a + a targeted recovery. The default value is none. Valid + values are none, immediate, + name, timestamp, + xid and lsn. + - Technically, this is a string parameter, but 'immediate' - is currently the only allowed value. + Target-type none specifies that recovery will not stop + until it runs out of WAL, which is the default setting. When not in targeted + recovery this is the only meaningful setting. - - - - recovery_target_name (string) - - recovery_target_name recovery parameter - - - - This parameter specifies the named restore point (created with - pg_create_restore_point()) to which recovery will proceed. + Target-type immediate specifies that recovery should end as + soon as a consistent state is reached, i.e. as early as possible. When restoring + from an online backup, this means the point where taking the backup ended. + For this target-type, no additional target-specifiers influence the stopping + point. - - - - recovery_target_time (timestamp) - - recovery_target_time recovery parameter - - - - This parameter specifies the time stamp up to which recovery - will proceed. + Target-type name specifies that recovery will + proceed to the point specified by recovery_target_value + when interpreted as the name of a restore point created with + pg_create_restore_point(). + The precise stopping point is also influenced by + and + . + + + + Target-type timestamp specifies that recovery will + proceed to the point specified by recovery_target_value + when interpreted as the timestamp of a transaction commit or abort. The precise stopping point is also influenced by - . + and + . - - - - recovery_target_xid (string) - - recovery_target_xid recovery parameter - - - - This parameter specifies the transaction ID up to which recovery - will proceed. Keep in mind + Target-type xid specifies that recovery will + proceed to the point specified by recovery_target_value + when interpreted as the transaction ID of a transaction commit or abort. + Keep in mind that while transaction IDs are assigned sequentially at transaction start, transactions can complete in a different numeric order. The transactions that will be recovered are those that committed before (and optionally including) the specified one. The precise stopping point is also influenced by - . + and + . + + + + Target-type lsn specifies that recovery will + proceed to the point specified by recovery_target_value + when interpreted as the LSN of any WAL record, parsed using the system + data type pg_lsn. + The precise stopping point is also influenced by + and + . - - recovery_target_lsn (pg_lsn) + + recovery_target_value (string) - recovery_target_lsn recovery parameter + recovery_target_value targeted recovery search parameter - This parameter specifies the LSN of the write-ahead log location up - to which recovery will proceed. The precise stopping point is also - influenced by . This - parameter is parsed using the system data type - pg_lsn. + Specifies the stopping point for targeted recovery. The string value + is interpreted according to strict rules according to the value of + recovery_target_type. An empty string may be an + invalid value in some cases. @@ -270,9 +293,9 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows Specifies whether to stop just after the specified recovery target (true), or just before the recovery target (false). - Applies when , - , or - is specified. + Applies when is specified + to lsn, timestamp or + xid. This setting controls whether transactions having exactly the target WAL location (LSN), commit time, or transaction ID, respectively, will be included in the recovery. Default is true. @@ -336,11 +359,11 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows since the last checkpoint next time it is started). - Note that because recovery.conf will not be renamed when - recovery_target_action is set to shutdown, + Note that because recovery.signal will not be + removed when recovery_target_action is set to shutdown, any subsequent start will end with immediate shutdown unless the - configuration is changed or the recovery.conf file is - removed manually. + configuration is changed or the recovery.signal + file is removed manually. This setting has no effect if no recovery target is set. @@ -358,24 +381,9 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows Standby Server Settings - - standby_mode (boolean) - - standby_mode recovery parameter - - - - - Specifies whether to start the PostgreSQL server as - a standby. If this parameter is on, the server will - not stop recovery when the end of archived WAL is reached, but - will keep trying to continue recovery by fetching new WAL segments - using restore_command - and/or by connecting to the primary server as specified by the - primary_conninfo setting. - - - + + New values for those parameters are considered only at restart of the server + primary_conninfo (string) @@ -385,7 +393,7 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows Specifies a connection string to be used for the standby server - to connect with the primary. This string is in the format + to connect with a sending server. This string is in the format described in . If any option is unspecified in this string, then the corresponding environment variable (see ) is checked. If the @@ -394,12 +402,12 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows The connection string should specify the host name (or address) - of the primary server, as well as the port number if it is not + of the sending server, as well as the port number if it is not the same as the standby server's default. Also specify a user name corresponding to a suitably-privileged role - on the primary (see + on the sending server (see ). - A password needs to be provided too, if the primary demands password + A password needs to be provided too, if the sender demands password authentication. It can be provided in the primary_conninfo string, or in a separate ~/.pgpass file on the standby server (use @@ -412,6 +420,7 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows + primary_slot_name (string) @@ -421,7 +430,7 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows Optionally specifies an existing replication slot to be used when - connecting to the primary via streaming replication to control + connecting to the sending server via streaming replication to control resource removal on the upstream node (see ). This setting has no effect if primary_conninfo is not @@ -429,10 +438,11 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows - - trigger_file (string) + + + promote_signal_file (string) - trigger_file recovery parameter + promote_signal_file recovery parameter @@ -440,7 +450,6 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows Specifies a trigger file whose presence ends recovery in the standby. Even if this value is not set, you can still promote the standby using pg_ctl promote. - This setting has no effect if standby_mode is off. @@ -454,7 +463,7 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows By default, a standby server restores WAL records from the - primary as soon as possible. It may be useful to have a time-delayed + sending server as soon as possible. It may be useful to have a time-delayed copy of the data, offering opportunities to correct data loss errors. This parameter allows you to delay recovery by a fixed period of time, measured in milliseconds if no unit is specified. For example, if diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml index c9f6ce4..91932b1 100644 --- a/doc/src/sgml/ref/pg_basebackup.sgml +++ b/doc/src/sgml/ref/pg_basebackup.sgml @@ -214,10 +214,11 @@ PostgreSQL documentation - Write a minimal recovery.conf in the output + Create standby.signal and append connection settings + to postgresql.auto.conf in the output directory (or into the base archive file when using tar format) to ease setting up a standby server. - The recovery.conf file will record the connection + The postgresql.auto.conf file will record the connection settings and, if specified, the replication slot that pg_basebackup is using, so that the streaming replication will use the same settings later on. @@ -470,7 +471,7 @@ PostgreSQL documentation replication slot. If the base backup is intended to be used as a streaming replication standby using replication slots, it should then use the same replication slot name - in recovery.conf. That way, it is ensured that + in . That way, it is ensured that the server does not remove any necessary WAL data in the time between the end of the base backup and the start of streaming replication. diff --git a/doc/src/sgml/ref/pg_rewind.sgml b/doc/src/sgml/ref/pg_rewind.sgml index e2662bb..a1cc24e 100644 --- a/doc/src/sgml/ref/pg_rewind.sgml +++ b/doc/src/sgml/ref/pg_rewind.sgml @@ -69,7 +69,8 @@ PostgreSQL documentation target cluster ran for a long time after the divergence, the old WAL files might no longer be present. In that case, they can be manually copied from the WAL archive to the pg_wal directory, or - fetched on startup by configuring recovery.conf. The use of + fetched on startup by configuring or + . The use of pg_rewind is not limited to failover, e.g. a standby server can be promoted, run some write transactions, and then rewinded to become a standby again. @@ -83,8 +84,9 @@ PostgreSQL documentation pg_rewind was run, and therefore could not be copied by the pg_rewind session, it must be made available when the target server is started. This can be done by creating a - recovery.conf file in the target data directory with a - suitable restore_command. + recovery.signal file in the target data directory + and configuring suitable + in postgresql.conf. diff --git a/doc/src/sgml/ref/pgarchivecleanup.sgml b/doc/src/sgml/ref/pgarchivecleanup.sgml index 4117a43..52674df 100644 --- a/doc/src/sgml/ref/pgarchivecleanup.sgml +++ b/doc/src/sgml/ref/pgarchivecleanup.sgml @@ -39,7 +39,7 @@ To configure a standby server to use pg_archivecleanup, put this into its - recovery.conf configuration file: + postgresql.conf configuration file: archive_cleanup_command = 'pg_archivecleanup archivelocation %r' diff --git a/doc/src/sgml/ref/pgupgrade.sgml b/doc/src/sgml/ref/pgupgrade.sgml index d51146d..51c044f 100644 --- a/doc/src/sgml/ref/pgupgrade.sgml +++ b/doc/src/sgml/ref/pgupgrade.sgml @@ -479,7 +479,7 @@ pg_upgrade.exe Save any configuration files from the old standbys' configuration directories you need to keep, e.g. postgresql.conf, - recovery.conf, because these will be overwritten or + pg_hba.conf, because these will be overwritten or removed in the next step. diff --git a/doc/src/sgml/release-9.1.sgml b/doc/src/sgml/release-9.1.sgml index e6ce800..cf41c75 100644 --- a/doc/src/sgml/release-9.1.sgml +++ b/doc/src/sgml/release-9.1.sgml @@ -9811,7 +9811,7 @@ Branch: REL9_0_STABLE [9d6af7367] 2015-08-15 11:02:34 -0400 These named restore points can be specified as recovery targets using the new recovery.conf setting - recovery_target_name. + recovery_target_name. diff --git a/doc/src/sgml/release-9.4.sgml b/doc/src/sgml/release-9.4.sgml index 6326610..0fa4291 100644 --- a/doc/src/sgml/release-9.4.sgml +++ b/doc/src/sgml/release-9.4.sgml @@ -11007,7 +11007,7 @@ Branch: REL9_4_STABLE [c2b06ab17] 2015-01-30 22:45:58 -0500 - Add + Add recovery_target option to stop WAL recovery as soon as a consistent state is reached (MauMau, Heikki Linnakangas) diff --git a/doc/src/sgml/release.sgml b/doc/src/sgml/release.sgml index c4e763a..4055adf 100644 --- a/doc/src/sgml/release.sgml +++ b/doc/src/sgml/release.sgml @@ -5,8 +5,7 @@ Typical markup: &<> use & escapes PostgreSQL -postgresql.conf, pg_hba.conf, - recovery.conf +postgresql.conf, pg_hba.conf [A-Z][A-Z_ ]+[A-Z_] , , , [A-Za-z_][A-Za-z0-9_]+() \-\-?[A-Za-z_]+[-A-Za-z_]*